diff options
Diffstat (limited to 'drivers/base')
117 files changed, 38150 insertions, 13126 deletions
diff --git a/drivers/base/Kconfig b/drivers/base/Kconfig index 5daa2599ed48..1786d87b29e2 100644 --- a/drivers/base/Kconfig +++ b/drivers/base/Kconfig @@ -1,10 +1,13 @@ +# SPDX-License-Identifier: GPL-2.0 menu "Generic Driver Options" -config UEVENT_HELPER_PATH - string "path to uevent helper" - default "" +config AUXILIARY_BUS + bool + +config UEVENT_HELPER + bool "Support for uevent helper" help - Path to uevent helper program forked by the kernel for + The uevent helper program is forked by the kernel for every uevent. Before the switch to the netlink-based uevent source, this was used to hook hotplug scripts into kernel device events. It @@ -15,8 +18,13 @@ config UEVENT_HELPER_PATH that it creates a high system load, or on smaller systems it is known to create out-of-memory situations during bootup. - To disable user space helper program execution at early boot - time specify an empty string here. This setting can be altered +config UEVENT_HELPER_PATH + string "path to uevent helper" + depends on UEVENT_HELPER + default "" + help + To disable user space helper program execution at by default + specify an empty string here. This setting can still be altered via /proc/sys/kernel/hotplug or via /sys/kernel/uevent_helper later at runtime. @@ -49,11 +57,22 @@ config DEVTMPFS_MOUNT with the commandline parameter: devtmpfs.mount=0|1. This option does not affect initramfs based booting, here the devtmpfs filesystem always needs to be mounted manually - after the roots is mounted. + after the rootfs is mounted. With this option enabled, it allows to bring up a system in rescue mode with init=/bin/sh, even when the /dev directory on the rootfs is completely empty. +config DEVTMPFS_SAFE + bool "Use nosuid,noexec mount options on devtmpfs" + depends on DEVTMPFS + help + This instructs the kernel to include the MS_NOEXEC and MS_NOSUID mount + flags when mounting devtmpfs. + + Notice: If enabled, things like /dev/mem cannot be mmapped + with the PROT_EXEC flag. This can break, for example, non-KMS + video drivers. + config STANDALONE bool "Select only drivers that don't need compile-time external firmware" default y @@ -64,95 +83,49 @@ config STANDALONE If unsure, say Y. config PREVENT_FIRMWARE_BUILD - bool "Prevent firmware from being built" + bool "Disable drivers features which enable custom firmware building" default y help - Say yes to avoid building firmware. Firmware is usually shipped - with the driver and only when updating the firmware should a - rebuild be made. - If unsure, say Y here. + Say yes to disable driver features which enable building a custom + driver firmware at kernel build time. These drivers do not use the + kernel firmware API to load firmware (CONFIG_FW_LOADER), instead they + use their own custom loading mechanism. The required firmware is + usually shipped with the driver, building the driver firmware + should only be needed if you have an updated firmware source. -config FW_LOADER - tristate "Userspace firmware loading support" if EXPERT - default y - ---help--- - This option is provided for the case where none of the in-tree modules - require userspace firmware loading support, but a module built - out-of-tree does. - -config FIRMWARE_IN_KERNEL - bool "Include in-kernel firmware blobs in kernel binary" - depends on FW_LOADER - default y - help - The kernel source tree includes a number of firmware 'blobs' - that are used by various drivers. The recommended way to - use these is to run "make firmware_install", which, after - converting ihex files to binary, copies all of the needed - binary files in firmware/ to /lib/firmware/ on your system so - that they can be loaded by userspace helpers on request. - - Enabling this option will build each required firmware blob - into the kernel directly, where request_firmware() will find - them without having to call out to userspace. This may be - useful if your root file system requires a device that uses - such firmware and do not wish to use an initrd. - - This single option controls the inclusion of firmware for - every driver that uses request_firmware() and ships its - firmware in the kernel source tree, which avoids a - proliferation of 'Include firmware for xxx device' options. - - Say 'N' and let firmware be loaded from userspace. - -config EXTRA_FIRMWARE - string "External firmware blobs to build into the kernel binary" - depends on FW_LOADER - help - This option allows firmware to be built into the kernel for the case - where the user either cannot or doesn't want to provide it from - userspace at runtime (for example, when the firmware in question is - required for accessing the boot device, and the user doesn't want to - use an initrd). - - This option is a string and takes the (space-separated) names of the - firmware files -- the same names that appear in MODULE_FIRMWARE() - and request_firmware() in the source. These files should exist under - the directory specified by the EXTRA_FIRMWARE_DIR option, which is - by default the firmware subdirectory of the kernel source tree. - - For example, you might set CONFIG_EXTRA_FIRMWARE="usb8388.bin", copy - the usb8388.bin file into the firmware directory, and build the kernel. - Then any request_firmware("usb8388.bin") will be satisfied internally - without needing to call out to userspace. - - WARNING: If you include additional firmware files into your binary - kernel image that are not available under the terms of the GPL, - then it may be a violation of the GPL to distribute the resulting - image since it combines both GPL and non-GPL work. You should - consult a lawyer of your own before distributing such an image. - -config EXTRA_FIRMWARE_DIR - string "Firmware blobs root directory" - depends on EXTRA_FIRMWARE != "" - default "firmware" + Firmware should not be being built as part of kernel, these days + you should always prevent this and say Y here. There are only two + old drivers which enable building of its firmware at kernel build + time: + + o CONFIG_WANXL through CONFIG_WANXL_BUILD_FIRMWARE + o CONFIG_SCSI_AIC79XX through CONFIG_AIC79XX_BUILD_FIRMWARE + +source "drivers/base/firmware_loader/Kconfig" + +config WANT_DEV_COREDUMP + bool help - This option controls the directory in which the kernel build system - looks for the firmware files listed in the EXTRA_FIRMWARE option. - The default is firmware/ in the kernel source tree, but by changing - this option you can point it elsewhere, such as /lib/firmware/ or - some other directory containing the firmware files. - -config FW_LOADER_USER_HELPER - bool "Fallback user-helper invocation for firmware loading" - depends on FW_LOADER + Drivers should "select" this option if they desire to use the + device coredump mechanism. + +config ALLOW_DEV_COREDUMP + bool "Allow device coredump" if EXPERT default y help - This option enables / disables the invocation of user-helper - (e.g. udev) for loading firmware files as a fallback after the - direct file loading in kernel fails. The user-mode helper is - no longer required unless you have a special firmware file that - resides in a non-standard path. + This option controls if the device coredump mechanism is available or + not; if disabled, the mechanism will be omitted even if drivers that + can use it are enabled. + Say 'N' for more sensitive systems or systems that don't want + to ever access the information to not have the code, nor keep any + data. + + If unsure, say Y. + +config DEV_COREDUMP + bool + default y if WANT_DEV_COREDUMP + depends on ALLOW_DEV_COREDUMP config DEBUG_DRIVER bool "Driver Core verbose debug messages" @@ -177,6 +150,39 @@ config DEBUG_DEVRES If you are unsure about this, Say N here. +config DEBUG_TEST_DRIVER_REMOVE + bool "Test driver remove calls during probe (UNSTABLE)" + depends on DEBUG_KERNEL + help + Say Y here if you want the Driver core to test driver remove functions + by calling probe, remove, probe. This tests the remove path without + having to unbind the driver or unload the driver module. + + This option is expected to find errors and may render your system + unusable. You should say N here unless you are explicitly looking to + test this functionality. + +config PM_QOS_KUNIT_TEST + bool "KUnit Test for PM QoS features" if !KUNIT_ALL_TESTS + depends on KUNIT=y + default KUNIT_ALL_TESTS + +config PM_RUNTIME_KUNIT_TEST + tristate "KUnit Tests for runtime PM" if !KUNIT_ALL_TESTS + depends on KUNIT + depends on PM + default KUNIT_ALL_TESTS + +config HMEM_REPORTING + bool + default n + depends on NUMA + help + Enable reporting for heterogeneous memory access attributes under + their non-uniform memory nodes. + +source "drivers/base/test/Kconfig" + config SYS_HYPERVISOR bool default n @@ -185,109 +191,62 @@ config GENERIC_CPU_DEVICES bool default n +config GENERIC_CPU_AUTOPROBE + bool + +config GENERIC_CPU_VULNERABILITIES + bool + config SOC_BUS bool + select GLOB source "drivers/base/regmap/Kconfig" config DMA_SHARED_BUFFER bool default n - select ANON_INODES + select IRQ_WORK help This option enables the framework for buffer-sharing between multiple drivers. A buffer is associated with a file using driver APIs extension; the file's descriptor can then be passed on to other driver. -config CMA - bool "Contiguous Memory Allocator" - depends on HAVE_DMA_CONTIGUOUS && HAVE_MEMBLOCK - select MIGRATION - select MEMORY_ISOLATION - help - This enables the Contiguous Memory Allocator which allows drivers - to allocate big physically-contiguous blocks of memory for use with - hardware components that do not support I/O map nor scatter-gather. - - For more information see <include/linux/dma-contiguous.h>. - If unsure, say "n". - -if CMA - -config CMA_DEBUG - bool "CMA debug messages (DEVELOPMENT)" - depends on DEBUG_KERNEL - help - Turns on debug messages in CMA. This produces KERN_DEBUG - messages for every CMA call as well as various messages while - processing calls such as dma_alloc_from_contiguous(). - This option does not affect warning and error messages. - -comment "Default contiguous memory area size:" - -config CMA_SIZE_MBYTES - int "Size in Mega Bytes" - depends on !CMA_SIZE_SEL_PERCENTAGE - default 16 - help - Defines the size (in MiB) of the default memory area for Contiguous - Memory Allocator. - -config CMA_SIZE_PERCENTAGE - int "Percentage of total memory" - depends on !CMA_SIZE_SEL_MBYTES - default 10 +config DMA_FENCE_TRACE + bool "Enable verbose DMA_FENCE_TRACE messages" + depends on DMA_SHARED_BUFFER help - Defines the size of the default memory area for Contiguous Memory - Allocator as a percentage of the total memory in the system. - -choice - prompt "Selected region size" - default CMA_SIZE_SEL_MBYTES - -config CMA_SIZE_SEL_MBYTES - bool "Use mega bytes value only" - -config CMA_SIZE_SEL_PERCENTAGE - bool "Use percentage value only" + Enable the DMA_FENCE_TRACE printks. This will add extra + spam to the console log, but will make it easier to diagnose + lockup related problems for dma-buffers shared across multiple + devices. -config CMA_SIZE_SEL_MIN - bool "Use lower value (minimum)" - -config CMA_SIZE_SEL_MAX - bool "Use higher value (maximum)" - -endchoice - -config CMA_ALIGNMENT - int "Maximum PAGE_SIZE order of alignment for contiguous buffers" - range 4 9 - default 8 - help - DMA mapping framework by default aligns all buffers to the smallest - PAGE_SIZE order which is greater than or equal to the requested buffer - size. This works well for buffers up to a few hundreds kilobytes, but - for larger buffers it just a memory waste. With this parameter you can - specify the maximum PAGE_SIZE order for contiguous buffers. Larger - buffers will be aligned only to this specified order. The order is - expressed as a power of two multiplied by the PAGE_SIZE. - - For example, if your system defaults to 4KiB pages, the order value - of 8 means that the buffers will be aligned up to 1MiB only. - - If unsure, leave the default value "8". - -config CMA_AREAS - int "Maximum count of the CMA device-private areas" - default 7 +config GENERIC_ARCH_TOPOLOGY + bool help - CMA allows to create CMA areas for particular devices. This parameter - sets the maximum number of such device private CMA areas in the - system. - - If unsure, leave the default value "7". + Enable support for architectures common topology code: e.g., parsing + CPU capacity information from DT, usage of such information for + appropriate scaling, sysfs interface for reading capacity values at + runtime. -endif +config GENERIC_ARCH_NUMA + bool + select NUMA_MEMBLKS + help + Enable support for generic NUMA implementation. Currently, RISC-V + and ARM64 use it. + +config FW_DEVLINK_SYNC_STATE_TIMEOUT + bool "sync_state() behavior defaults to timeout instead of strict" + help + This is build time equivalent of adding kernel command line parameter + "fw_devlink.sync_state=timeout". Give up waiting on consumers and + call sync_state() on any devices that haven't yet received their + sync_state() calls after deferred_probe_timeout has expired or by + late_initcall() if !CONFIG_MODULES. You should almost always want to + select N here unless you have already successfully tested with the + command line option on every system/board your kernel is expected to + work on. endmenu diff --git a/drivers/base/Makefile b/drivers/base/Makefile index 48029aa477d9..8074a10183dc 100644 --- a/drivers/base/Makefile +++ b/drivers/base/Makefile @@ -1,27 +1,37 @@ +# SPDX-License-Identifier: GPL-2.0 # Makefile for the Linux device tree -obj-y := core.o bus.o dd.o syscore.o \ +obj-y := component.o core.o bus.o dd.o syscore.o \ driver.o class.o platform.o \ cpu.o firmware.o init.o map.o devres.o \ attribute_container.o transport_class.o \ - topology.o + topology.o container.o property.o cacheinfo.o \ + swnode.o faux.o +obj-$(CONFIG_AUXILIARY_BUS) += auxiliary.o obj-$(CONFIG_DEVTMPFS) += devtmpfs.o -obj-$(CONFIG_CMA) += dma-contiguous.o obj-y += power/ -obj-$(CONFIG_HAS_DMA) += dma-mapping.o -obj-$(CONFIG_HAVE_GENERIC_DMA_COHERENT) += dma-coherent.o -obj-$(CONFIG_DMA_SHARED_BUFFER) += dma-buf.o reservation.o -obj-$(CONFIG_ISA) += isa.o -obj-$(CONFIG_FW_LOADER) += firmware_class.o +obj-$(CONFIG_ISA_BUS_API) += isa.o +obj-y += firmware_loader/ obj-$(CONFIG_NUMA) += node.o -obj-$(CONFIG_MEMORY_HOTPLUG_SPARSE) += memory.o +obj-$(CONFIG_MEMORY_HOTPLUG) += memory.o ifeq ($(CONFIG_SYSFS),y) obj-$(CONFIG_MODULES) += module.o +obj-$(CONFIG_AUXILIARY_BUS) += auxiliary_sysfs.o endif obj-$(CONFIG_SYS_HYPERVISOR) += hypervisor.o obj-$(CONFIG_REGMAP) += regmap/ obj-$(CONFIG_SOC_BUS) += soc.o obj-$(CONFIG_PINCTRL) += pinctrl.o +obj-$(CONFIG_DEV_COREDUMP) += devcoredump.o +obj-$(CONFIG_GENERIC_MSI_IRQ) += platform-msi.o +obj-$(CONFIG_GENERIC_ARCH_TOPOLOGY) += arch_topology.o +obj-$(CONFIG_GENERIC_ARCH_NUMA) += arch_numa.o +obj-$(CONFIG_ACPI) += physical_location.o + +obj-y += test/ ccflags-$(CONFIG_DEBUG_DRIVER) := -DDEBUG +# define_trace.h needs to know how to find our header +CFLAGS_trace.o := -I$(src) +obj-$(CONFIG_TRACING) += trace.o diff --git a/drivers/base/arch_numa.c b/drivers/base/arch_numa.c new file mode 100644 index 000000000000..c99f2ab105e5 --- /dev/null +++ b/drivers/base/arch_numa.c @@ -0,0 +1,379 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * NUMA support, based on the x86 implementation. + * + * Copyright (C) 2015 Cavium Inc. + * Author: Ganapatrao Kulkarni <gkulkarni@cavium.com> + */ + +#define pr_fmt(fmt) "NUMA: " fmt + +#include <linux/acpi.h> +#include <linux/memblock.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/numa_memblks.h> + +#include <asm/sections.h> + +static int cpu_to_node_map[NR_CPUS] = { [0 ... NR_CPUS-1] = NUMA_NO_NODE }; + +bool numa_off; + +static __init int numa_parse_early_param(char *opt) +{ + if (!opt) + return -EINVAL; + if (str_has_prefix(opt, "off")) + numa_off = true; + if (!strncmp(opt, "fake=", 5)) + return numa_emu_cmdline(opt + 5); + + return 0; +} +early_param("numa", numa_parse_early_param); + +cpumask_var_t node_to_cpumask_map[MAX_NUMNODES]; +EXPORT_SYMBOL(node_to_cpumask_map); + +#ifdef CONFIG_DEBUG_PER_CPU_MAPS + +/* + * Returns a pointer to the bitmask of CPUs on Node 'node'. + */ +const struct cpumask *cpumask_of_node(int node) +{ + + if (node == NUMA_NO_NODE) + return cpu_all_mask; + + if (WARN_ON(node < 0 || node >= nr_node_ids)) + return cpu_none_mask; + + if (WARN_ON(node_to_cpumask_map[node] == NULL)) + return cpu_online_mask; + + return node_to_cpumask_map[node]; +} +EXPORT_SYMBOL(cpumask_of_node); + +#endif + +#ifndef CONFIG_NUMA_EMU +static void numa_update_cpu(unsigned int cpu, bool remove) +{ + int nid = cpu_to_node(cpu); + + if (nid == NUMA_NO_NODE) + return; + + if (remove) + cpumask_clear_cpu(cpu, node_to_cpumask_map[nid]); + else + cpumask_set_cpu(cpu, node_to_cpumask_map[nid]); +} + +void numa_add_cpu(unsigned int cpu) +{ + numa_update_cpu(cpu, false); +} + +void numa_remove_cpu(unsigned int cpu) +{ + numa_update_cpu(cpu, true); +} +#endif + +void numa_clear_node(unsigned int cpu) +{ + numa_remove_cpu(cpu); + set_cpu_numa_node(cpu, NUMA_NO_NODE); +} + +/* + * Allocate node_to_cpumask_map based on number of available nodes + * Requires node_possible_map to be valid. + * + * Note: cpumask_of_node() is not valid until after this is done. + * (Use CONFIG_DEBUG_PER_CPU_MAPS to check this.) + */ +static void __init setup_node_to_cpumask_map(void) +{ + int node; + + /* setup nr_node_ids if not done yet */ + if (nr_node_ids == MAX_NUMNODES) + setup_nr_node_ids(); + + /* allocate and clear the mapping */ + for (node = 0; node < nr_node_ids; node++) { + alloc_bootmem_cpumask_var(&node_to_cpumask_map[node]); + cpumask_clear(node_to_cpumask_map[node]); + } + + /* cpumask_of_node() will now work */ + pr_debug("Node to cpumask map for %u nodes\n", nr_node_ids); +} + +/* + * Set the cpu to node and mem mapping + */ +void numa_store_cpu_info(unsigned int cpu) +{ + set_cpu_numa_node(cpu, cpu_to_node_map[cpu]); +} + +void __init early_map_cpu_to_node(unsigned int cpu, int nid) +{ + /* fallback to node 0 */ + if (nid < 0 || nid >= MAX_NUMNODES || numa_off) + nid = 0; + + cpu_to_node_map[cpu] = nid; + + /* + * We should set the numa node of cpu0 as soon as possible, because it + * has already been set up online before. cpu_to_node(0) will soon be + * called. + */ + if (!cpu) + set_cpu_numa_node(cpu, nid); +} + +#ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA +unsigned long __per_cpu_offset[NR_CPUS] __read_mostly; +EXPORT_SYMBOL(__per_cpu_offset); + +int early_cpu_to_node(int cpu) +{ + return cpu_to_node_map[cpu]; +} + +static int __init pcpu_cpu_distance(unsigned int from, unsigned int to) +{ + return node_distance(early_cpu_to_node(from), early_cpu_to_node(to)); +} + +void __init setup_per_cpu_areas(void) +{ + unsigned long delta; + unsigned int cpu; + int rc = -EINVAL; + + if (pcpu_chosen_fc != PCPU_FC_PAGE) { + /* + * Always reserve area for module percpu variables. That's + * what the legacy allocator did. + */ + rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE, + PERCPU_DYNAMIC_RESERVE, PAGE_SIZE, + pcpu_cpu_distance, + early_cpu_to_node); +#ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK + if (rc < 0) + pr_warn("PERCPU: %s allocator failed (%d), falling back to page size\n", + pcpu_fc_names[pcpu_chosen_fc], rc); +#endif + } + +#ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK + if (rc < 0) + rc = pcpu_page_first_chunk(PERCPU_MODULE_RESERVE, early_cpu_to_node); +#endif + if (rc < 0) + panic("Failed to initialize percpu areas (err=%d).", rc); + + delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start; + for_each_possible_cpu(cpu) + __per_cpu_offset[cpu] = delta + pcpu_unit_offsets[cpu]; +} +#endif + +/* + * Initialize NODE_DATA for a node on the local memory + */ +static void __init setup_node_data(int nid, u64 start_pfn, u64 end_pfn) +{ + if (start_pfn >= end_pfn) + pr_info("Initmem setup node %d [<memory-less node>]\n", nid); + + alloc_node_data(nid); + + NODE_DATA(nid)->node_id = nid; + NODE_DATA(nid)->node_start_pfn = start_pfn; + NODE_DATA(nid)->node_spanned_pages = end_pfn - start_pfn; +} + +static int __init numa_register_nodes(void) +{ + int nid; + + /* Check the validity of the memblock/node mapping */ + if (!memblock_validate_numa_coverage(0)) + return -EINVAL; + + /* Finally register nodes. */ + for_each_node_mask(nid, numa_nodes_parsed) { + unsigned long start_pfn, end_pfn; + + get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); + setup_node_data(nid, start_pfn, end_pfn); + node_set_online(nid); + } + + /* Setup online nodes to actual nodes*/ + node_possible_map = numa_nodes_parsed; + + return 0; +} + +static int __init numa_init(int (*init_func)(void)) +{ + int ret; + + nodes_clear(numa_nodes_parsed); + nodes_clear(node_possible_map); + nodes_clear(node_online_map); + + ret = numa_memblks_init(init_func, /* memblock_force_top_down */ false); + if (ret < 0) + goto out_free_distance; + + if (nodes_empty(numa_nodes_parsed)) { + pr_info("No NUMA configuration found\n"); + ret = -EINVAL; + goto out_free_distance; + } + + ret = numa_register_nodes(); + if (ret < 0) + goto out_free_distance; + + setup_node_to_cpumask_map(); + + return 0; +out_free_distance: + numa_reset_distance(); + return ret; +} + +/** + * dummy_numa_init() - Fallback dummy NUMA init + * + * Used if there's no underlying NUMA architecture, NUMA initialization + * fails, or NUMA is disabled on the command line. + * + * Must online at least one node (node 0) and add memory blocks that cover all + * allowed memory. It is unlikely that this function fails. + * + * Return: 0 on success, -errno on failure. + */ +static int __init dummy_numa_init(void) +{ + phys_addr_t start = memblock_start_of_DRAM(); + phys_addr_t end = memblock_end_of_DRAM() - 1; + int ret; + + if (numa_off) + pr_info("NUMA disabled\n"); /* Forced off on command line. */ + pr_info("Faking a node at [mem %pap-%pap]\n", &start, &end); + + ret = numa_add_memblk(0, start, end + 1); + if (ret) { + pr_err("NUMA init failed\n"); + return ret; + } + node_set(0, numa_nodes_parsed); + + numa_off = true; + return 0; +} + +#ifdef CONFIG_ACPI_NUMA +static int __init arch_acpi_numa_init(void) +{ + int ret; + + ret = acpi_numa_init(); + if (ret) { + pr_debug("Failed to initialise from firmware\n"); + return ret; + } + + return srat_disabled() ? -EINVAL : 0; +} +#else +static int __init arch_acpi_numa_init(void) +{ + return -EOPNOTSUPP; +} +#endif + +/** + * arch_numa_init() - Initialize NUMA + * + * Try each configured NUMA initialization method until one succeeds. The + * last fallback is dummy single node config encompassing whole memory. + */ +void __init arch_numa_init(void) +{ + if (!numa_off) { + if (!acpi_disabled && !numa_init(arch_acpi_numa_init)) + return; + if (acpi_disabled && !numa_init(of_numa_init)) + return; + } + + numa_init(dummy_numa_init); +} + +#ifdef CONFIG_NUMA_EMU +void __init numa_emu_update_cpu_to_node(int *emu_nid_to_phys, + unsigned int nr_emu_nids) +{ + int i, j; + + /* + * Transform cpu_to_node_map table to use emulated nids by + * reverse-mapping phys_nid. The maps should always exist but fall + * back to zero just in case. + */ + for (i = 0; i < ARRAY_SIZE(cpu_to_node_map); i++) { + if (cpu_to_node_map[i] == NUMA_NO_NODE) + continue; + for (j = 0; j < nr_emu_nids; j++) + if (cpu_to_node_map[i] == emu_nid_to_phys[j]) + break; + cpu_to_node_map[i] = j < nr_emu_nids ? j : 0; + } +} + +u64 __init numa_emu_dma_end(void) +{ + return memblock_start_of_DRAM() + SZ_4G; +} + +void debug_cpumask_set_cpu(unsigned int cpu, int node, bool enable) +{ + struct cpumask *mask; + + if (node == NUMA_NO_NODE) + return; + + mask = node_to_cpumask_map[node]; + if (!cpumask_available(mask)) { + pr_err("node_to_cpumask_map[%i] NULL\n", node); + dump_stack(); + return; + } + + if (enable) + cpumask_set_cpu(cpu, mask); + else + cpumask_clear_cpu(cpu, mask); + + pr_debug("%s cpu %d node %d: mask now %*pbl\n", + enable ? "numa_add_cpu" : "numa_remove_cpu", + cpu, node, cpumask_pr_args(mask)); +} +#endif /* CONFIG_NUMA_EMU */ diff --git a/drivers/base/arch_topology.c b/drivers/base/arch_topology.c new file mode 100644 index 000000000000..84ec92bff642 --- /dev/null +++ b/drivers/base/arch_topology.c @@ -0,0 +1,972 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Arch specific cpu topology information + * + * Copyright (C) 2016, ARM Ltd. + * Written by: Juri Lelli, ARM Ltd. + */ + +#include <linux/acpi.h> +#include <linux/cacheinfo.h> +#include <linux/cleanup.h> +#include <linux/cpu.h> +#include <linux/cpufreq.h> +#include <linux/cpu_smt.h> +#include <linux/device.h> +#include <linux/of.h> +#include <linux/slab.h> +#include <linux/sched/topology.h> +#include <linux/cpuset.h> +#include <linux/cpumask.h> +#include <linux/init.h> +#include <linux/rcupdate.h> +#include <linux/sched.h> +#include <linux/units.h> + +#define CREATE_TRACE_POINTS +#include <trace/events/hw_pressure.h> + +static DEFINE_PER_CPU(struct scale_freq_data __rcu *, sft_data); +static struct cpumask scale_freq_counters_mask; +static bool scale_freq_invariant; +DEFINE_PER_CPU(unsigned long, capacity_freq_ref) = 0; +EXPORT_PER_CPU_SYMBOL_GPL(capacity_freq_ref); + +static bool supports_scale_freq_counters(const struct cpumask *cpus) +{ + return cpumask_subset(cpus, &scale_freq_counters_mask); +} + +bool topology_scale_freq_invariant(void) +{ + return cpufreq_supports_freq_invariance() || + supports_scale_freq_counters(cpu_online_mask); +} + +static void update_scale_freq_invariant(bool status) +{ + if (scale_freq_invariant == status) + return; + + /* + * Task scheduler behavior depends on frequency invariance support, + * either cpufreq or counter driven. If the support status changes as + * a result of counter initialisation and use, retrigger the build of + * scheduling domains to ensure the information is propagated properly. + */ + if (topology_scale_freq_invariant() == status) { + scale_freq_invariant = status; + rebuild_sched_domains_energy(); + } +} + +void topology_set_scale_freq_source(struct scale_freq_data *data, + const struct cpumask *cpus) +{ + struct scale_freq_data *sfd; + int cpu; + + /* + * Avoid calling rebuild_sched_domains() unnecessarily if FIE is + * supported by cpufreq. + */ + if (cpumask_empty(&scale_freq_counters_mask)) + scale_freq_invariant = topology_scale_freq_invariant(); + + rcu_read_lock(); + + for_each_cpu(cpu, cpus) { + sfd = rcu_dereference(*per_cpu_ptr(&sft_data, cpu)); + + /* Use ARCH provided counters whenever possible */ + if (!sfd || sfd->source != SCALE_FREQ_SOURCE_ARCH) { + rcu_assign_pointer(per_cpu(sft_data, cpu), data); + cpumask_set_cpu(cpu, &scale_freq_counters_mask); + } + } + + rcu_read_unlock(); + + update_scale_freq_invariant(true); +} +EXPORT_SYMBOL_GPL(topology_set_scale_freq_source); + +void topology_clear_scale_freq_source(enum scale_freq_source source, + const struct cpumask *cpus) +{ + struct scale_freq_data *sfd; + int cpu; + + rcu_read_lock(); + + for_each_cpu(cpu, cpus) { + sfd = rcu_dereference(*per_cpu_ptr(&sft_data, cpu)); + + if (sfd && sfd->source == source) { + rcu_assign_pointer(per_cpu(sft_data, cpu), NULL); + cpumask_clear_cpu(cpu, &scale_freq_counters_mask); + } + } + + rcu_read_unlock(); + + /* + * Make sure all references to previous sft_data are dropped to avoid + * use-after-free races. + */ + synchronize_rcu(); + + update_scale_freq_invariant(false); +} +EXPORT_SYMBOL_GPL(topology_clear_scale_freq_source); + +void topology_scale_freq_tick(void) +{ + struct scale_freq_data *sfd = rcu_dereference_sched(*this_cpu_ptr(&sft_data)); + + if (sfd) + sfd->set_freq_scale(); +} + +DEFINE_PER_CPU(unsigned long, arch_freq_scale) = SCHED_CAPACITY_SCALE; +EXPORT_PER_CPU_SYMBOL_GPL(arch_freq_scale); + +void topology_set_freq_scale(const struct cpumask *cpus, unsigned long cur_freq, + unsigned long max_freq) +{ + unsigned long scale; + int i; + + if (WARN_ON_ONCE(!cur_freq || !max_freq)) + return; + + /* + * If the use of counters for FIE is enabled, just return as we don't + * want to update the scale factor with information from CPUFREQ. + * Instead the scale factor will be updated from arch_scale_freq_tick. + */ + if (supports_scale_freq_counters(cpus)) + return; + + scale = (cur_freq << SCHED_CAPACITY_SHIFT) / max_freq; + + for_each_cpu(i, cpus) + per_cpu(arch_freq_scale, i) = scale; +} + +DEFINE_PER_CPU(unsigned long, hw_pressure); + +/** + * topology_update_hw_pressure() - Update HW pressure for CPUs + * @cpus : The related CPUs for which capacity has been reduced + * @capped_freq : The maximum allowed frequency that CPUs can run at + * + * Update the value of HW pressure for all @cpus in the mask. The + * cpumask should include all (online+offline) affected CPUs, to avoid + * operating on stale data when hot-plug is used for some CPUs. The + * @capped_freq reflects the currently allowed max CPUs frequency due to + * HW capping. It might be also a boost frequency value, which is bigger + * than the internal 'capacity_freq_ref' max frequency. In such case the + * pressure value should simply be removed, since this is an indication that + * there is no HW throttling. The @capped_freq must be provided in kHz. + */ +void topology_update_hw_pressure(const struct cpumask *cpus, + unsigned long capped_freq) +{ + unsigned long max_capacity, capacity, pressure; + u32 max_freq; + int cpu; + + cpu = cpumask_first(cpus); + max_capacity = arch_scale_cpu_capacity(cpu); + max_freq = arch_scale_freq_ref(cpu); + + /* + * Handle properly the boost frequencies, which should simply clean + * the HW pressure value. + */ + if (max_freq <= capped_freq) + capacity = max_capacity; + else + capacity = mult_frac(max_capacity, capped_freq, max_freq); + + pressure = max_capacity - capacity; + + trace_hw_pressure_update(cpu, pressure); + + for_each_cpu(cpu, cpus) + WRITE_ONCE(per_cpu(hw_pressure, cpu), pressure); +} +EXPORT_SYMBOL_GPL(topology_update_hw_pressure); + +static void update_topology_flags_workfn(struct work_struct *work); +static DECLARE_WORK(update_topology_flags_work, update_topology_flags_workfn); + +static int update_topology; + +int topology_update_cpu_topology(void) +{ + return update_topology; +} + +/* + * Updating the sched_domains can't be done directly from cpufreq callbacks + * due to locking, so queue the work for later. + */ +static void update_topology_flags_workfn(struct work_struct *work) +{ + update_topology = 1; + rebuild_sched_domains(); + pr_debug("sched_domain hierarchy rebuilt, flags updated\n"); + update_topology = 0; +} + +static u32 *raw_capacity; + +static int free_raw_capacity(void) +{ + kfree(raw_capacity); + raw_capacity = NULL; + + return 0; +} + +void topology_normalize_cpu_scale(void) +{ + u64 capacity; + u64 capacity_scale; + int cpu; + + if (!raw_capacity) + return; + + capacity_scale = 1; + for_each_possible_cpu(cpu) { + capacity = raw_capacity[cpu] * + (per_cpu(capacity_freq_ref, cpu) ?: 1); + capacity_scale = max(capacity, capacity_scale); + } + + pr_debug("cpu_capacity: capacity_scale=%llu\n", capacity_scale); + for_each_possible_cpu(cpu) { + capacity = raw_capacity[cpu] * + (per_cpu(capacity_freq_ref, cpu) ?: 1); + capacity = div64_u64(capacity << SCHED_CAPACITY_SHIFT, + capacity_scale); + topology_set_cpu_scale(cpu, capacity); + pr_debug("cpu_capacity: CPU%d cpu_capacity=%lu\n", + cpu, topology_get_cpu_scale(cpu)); + } +} + +bool __init topology_parse_cpu_capacity(struct device_node *cpu_node, int cpu) +{ + struct clk *cpu_clk; + static bool cap_parsing_failed; + int ret; + u32 cpu_capacity; + + if (cap_parsing_failed) + return false; + + ret = of_property_read_u32(cpu_node, "capacity-dmips-mhz", + &cpu_capacity); + if (!ret) { + if (!raw_capacity) { + raw_capacity = kcalloc(num_possible_cpus(), + sizeof(*raw_capacity), + GFP_KERNEL); + if (!raw_capacity) { + cap_parsing_failed = true; + return false; + } + } + raw_capacity[cpu] = cpu_capacity; + pr_debug("cpu_capacity: %pOF cpu_capacity=%u (raw)\n", + cpu_node, raw_capacity[cpu]); + + /* + * Update capacity_freq_ref for calculating early boot CPU capacities. + * For non-clk CPU DVFS mechanism, there's no way to get the + * frequency value now, assuming they are running at the same + * frequency (by keeping the initial capacity_freq_ref value). + */ + cpu_clk = of_clk_get(cpu_node, 0); + if (!IS_ERR_OR_NULL(cpu_clk)) { + per_cpu(capacity_freq_ref, cpu) = + clk_get_rate(cpu_clk) / HZ_PER_KHZ; + clk_put(cpu_clk); + } + } else { + if (raw_capacity) { + pr_err("cpu_capacity: missing %pOF raw capacity\n", + cpu_node); + pr_err("cpu_capacity: partial information: fallback to 1024 for all CPUs\n"); + } + cap_parsing_failed = true; + free_raw_capacity(); + } + + return !ret; +} + +void __weak freq_inv_set_max_ratio(int cpu, u64 max_rate) +{ +} + +#ifdef CONFIG_ACPI_CPPC_LIB +#include <acpi/cppc_acpi.h> + +static inline void topology_init_cpu_capacity_cppc(void) +{ + u64 capacity, capacity_scale = 0; + struct cppc_perf_caps perf_caps; + int cpu; + + if (likely(!acpi_cpc_valid())) + return; + + raw_capacity = kcalloc(num_possible_cpus(), sizeof(*raw_capacity), + GFP_KERNEL); + if (!raw_capacity) + return; + + for_each_possible_cpu(cpu) { + if (!cppc_get_perf_caps(cpu, &perf_caps) && + (perf_caps.highest_perf >= perf_caps.nominal_perf) && + (perf_caps.highest_perf >= perf_caps.lowest_perf)) { + raw_capacity[cpu] = perf_caps.highest_perf; + capacity_scale = max_t(u64, capacity_scale, raw_capacity[cpu]); + + per_cpu(capacity_freq_ref, cpu) = cppc_perf_to_khz(&perf_caps, raw_capacity[cpu]); + + pr_debug("cpu_capacity: CPU%d cpu_capacity=%u (raw).\n", + cpu, raw_capacity[cpu]); + continue; + } + + pr_err("cpu_capacity: CPU%d missing/invalid highest performance.\n", cpu); + pr_err("cpu_capacity: partial information: fallback to 1024 for all CPUs\n"); + goto exit; + } + + for_each_possible_cpu(cpu) { + freq_inv_set_max_ratio(cpu, + per_cpu(capacity_freq_ref, cpu) * HZ_PER_KHZ); + + capacity = raw_capacity[cpu]; + capacity = div64_u64(capacity << SCHED_CAPACITY_SHIFT, + capacity_scale); + topology_set_cpu_scale(cpu, capacity); + pr_debug("cpu_capacity: CPU%d cpu_capacity=%lu\n", + cpu, topology_get_cpu_scale(cpu)); + } + + schedule_work(&update_topology_flags_work); + pr_debug("cpu_capacity: cpu_capacity initialization done\n"); + +exit: + free_raw_capacity(); +} +void acpi_processor_init_invariance_cppc(void) +{ + topology_init_cpu_capacity_cppc(); +} +#endif + +#ifdef CONFIG_CPU_FREQ +static cpumask_var_t cpus_to_visit; +static void parsing_done_workfn(struct work_struct *work); +static DECLARE_WORK(parsing_done_work, parsing_done_workfn); + +static int +init_cpu_capacity_callback(struct notifier_block *nb, + unsigned long val, + void *data) +{ + struct cpufreq_policy *policy = data; + int cpu; + + if (val != CPUFREQ_CREATE_POLICY) + return 0; + + pr_debug("cpu_capacity: init cpu capacity for CPUs [%*pbl] (to_visit=%*pbl)\n", + cpumask_pr_args(policy->related_cpus), + cpumask_pr_args(cpus_to_visit)); + + cpumask_andnot(cpus_to_visit, cpus_to_visit, policy->related_cpus); + + for_each_cpu(cpu, policy->related_cpus) { + per_cpu(capacity_freq_ref, cpu) = policy->cpuinfo.max_freq; + freq_inv_set_max_ratio(cpu, + per_cpu(capacity_freq_ref, cpu) * HZ_PER_KHZ); + } + + if (cpumask_empty(cpus_to_visit)) { + if (raw_capacity) { + topology_normalize_cpu_scale(); + schedule_work(&update_topology_flags_work); + free_raw_capacity(); + } + pr_debug("cpu_capacity: parsing done\n"); + schedule_work(&parsing_done_work); + } + + return 0; +} + +static struct notifier_block init_cpu_capacity_notifier = { + .notifier_call = init_cpu_capacity_callback, +}; + +static int __init register_cpufreq_notifier(void) +{ + int ret; + + /* + * On ACPI-based systems skip registering cpufreq notifier as cpufreq + * information is not needed for cpu capacity initialization. + */ + if (!acpi_disabled) + return -EINVAL; + + if (!alloc_cpumask_var(&cpus_to_visit, GFP_KERNEL)) + return -ENOMEM; + + cpumask_copy(cpus_to_visit, cpu_possible_mask); + + ret = cpufreq_register_notifier(&init_cpu_capacity_notifier, + CPUFREQ_POLICY_NOTIFIER); + + if (ret) + free_cpumask_var(cpus_to_visit); + + return ret; +} +core_initcall(register_cpufreq_notifier); + +static void parsing_done_workfn(struct work_struct *work) +{ + cpufreq_unregister_notifier(&init_cpu_capacity_notifier, + CPUFREQ_POLICY_NOTIFIER); + free_cpumask_var(cpus_to_visit); +} + +#else +core_initcall(free_raw_capacity); +#endif + +#if defined(CONFIG_ARM64) || defined(CONFIG_RISCV) + +/* Used to enable the SMT control */ +static unsigned int max_smt_thread_num = 1; + +/* + * This function returns the logic cpu number of the node. + * There are basically three kinds of return values: + * (1) logic cpu number which is > 0. + * (2) -ENODEV when the device tree(DT) node is valid and found in the DT but + * there is no possible logical CPU in the kernel to match. This happens + * when CONFIG_NR_CPUS is configure to be smaller than the number of + * CPU nodes in DT. We need to just ignore this case. + * (3) -1 if the node does not exist in the device tree + */ +static int __init get_cpu_for_node(struct device_node *node) +{ + int cpu; + struct device_node *cpu_node __free(device_node) = + of_parse_phandle(node, "cpu", 0); + + if (!cpu_node) + return -1; + + cpu = of_cpu_node_to_id(cpu_node); + if (cpu >= 0) + topology_parse_cpu_capacity(cpu_node, cpu); + else + pr_info("CPU node for %pOF exist but the possible cpu range is :%*pbl\n", + cpu_node, cpumask_pr_args(cpu_possible_mask)); + + return cpu; +} + +static int __init parse_core(struct device_node *core, int package_id, + int cluster_id, int core_id) +{ + char name[20]; + bool leaf = true; + int i = 0; + int cpu; + + do { + snprintf(name, sizeof(name), "thread%d", i); + struct device_node *t __free(device_node) = + of_get_child_by_name(core, name); + + if (!t) + break; + + leaf = false; + cpu = get_cpu_for_node(t); + if (cpu >= 0) { + cpu_topology[cpu].package_id = package_id; + cpu_topology[cpu].cluster_id = cluster_id; + cpu_topology[cpu].core_id = core_id; + cpu_topology[cpu].thread_id = i; + } else if (cpu != -ENODEV) { + pr_err("%pOF: Can't get CPU for thread\n", t); + return -EINVAL; + } + i++; + } while (1); + + max_smt_thread_num = max_t(unsigned int, max_smt_thread_num, i); + + cpu = get_cpu_for_node(core); + if (cpu >= 0) { + if (!leaf) { + pr_err("%pOF: Core has both threads and CPU\n", + core); + return -EINVAL; + } + + cpu_topology[cpu].package_id = package_id; + cpu_topology[cpu].cluster_id = cluster_id; + cpu_topology[cpu].core_id = core_id; + } else if (leaf && cpu != -ENODEV) { + pr_err("%pOF: Can't get CPU for leaf core\n", core); + return -EINVAL; + } + + return 0; +} + +static int __init parse_cluster(struct device_node *cluster, int package_id, + int cluster_id, int depth) +{ + char name[20]; + bool leaf = true; + bool has_cores = false; + int core_id = 0; + int i, ret; + + /* + * First check for child clusters; we currently ignore any + * information about the nesting of clusters and present the + * scheduler with a flat list of them. + */ + i = 0; + do { + snprintf(name, sizeof(name), "cluster%d", i); + struct device_node *c __free(device_node) = + of_get_child_by_name(cluster, name); + + if (!c) + break; + + leaf = false; + ret = parse_cluster(c, package_id, i, depth + 1); + if (depth > 0) + pr_warn("Topology for clusters of clusters not yet supported\n"); + if (ret != 0) + return ret; + i++; + } while (1); + + /* Now check for cores */ + i = 0; + do { + snprintf(name, sizeof(name), "core%d", i); + struct device_node *c __free(device_node) = + of_get_child_by_name(cluster, name); + + if (!c) + break; + + has_cores = true; + + if (depth == 0) { + pr_err("%pOF: cpu-map children should be clusters\n", c); + return -EINVAL; + } + + if (leaf) { + ret = parse_core(c, package_id, cluster_id, core_id++); + if (ret != 0) + return ret; + } else { + pr_err("%pOF: Non-leaf cluster with core %s\n", + cluster, name); + return -EINVAL; + } + + i++; + } while (1); + + if (leaf && !has_cores) + pr_warn("%pOF: empty cluster\n", cluster); + + return 0; +} + +static int __init parse_socket(struct device_node *socket) +{ + char name[20]; + bool has_socket = false; + int package_id = 0, ret; + + do { + snprintf(name, sizeof(name), "socket%d", package_id); + struct device_node *c __free(device_node) = + of_get_child_by_name(socket, name); + + if (!c) + break; + + has_socket = true; + ret = parse_cluster(c, package_id, -1, 0); + if (ret != 0) + return ret; + + package_id++; + } while (1); + + if (!has_socket) + ret = parse_cluster(socket, 0, -1, 0); + + /* + * Reset the max_smt_thread_num to 1 on failure. Since on failure + * we need to notify the framework the SMT is not supported, but + * max_smt_thread_num can be initialized to the SMT thread number + * of the cores which are successfully parsed. + */ + if (ret) + max_smt_thread_num = 1; + + cpu_smt_set_num_threads(max_smt_thread_num, max_smt_thread_num); + + return ret; +} + +static int __init parse_dt_topology(void) +{ + int ret = 0; + int cpu; + struct device_node *cn __free(device_node) = + of_find_node_by_path("/cpus"); + + if (!cn) { + pr_err("No CPU information found in DT\n"); + return 0; + } + + /* + * When topology is provided cpu-map is essentially a root + * cluster with restricted subnodes. + */ + struct device_node *map __free(device_node) = + of_get_child_by_name(cn, "cpu-map"); + + if (!map) + return ret; + + ret = parse_socket(map); + if (ret != 0) + return ret; + + topology_normalize_cpu_scale(); + + /* + * Check that all cores are in the topology; the SMP code will + * only mark cores described in the DT as possible. + */ + for_each_possible_cpu(cpu) + if (cpu_topology[cpu].package_id < 0) { + return -EINVAL; + } + + return ret; +} +#endif + +/* + * cpu topology table + */ +struct cpu_topology cpu_topology[NR_CPUS]; +EXPORT_SYMBOL_GPL(cpu_topology); + +const struct cpumask *cpu_coregroup_mask(int cpu) +{ + const cpumask_t *core_mask = cpumask_of_node(cpu_to_node(cpu)); + + /* Find the smaller of NUMA, core or LLC siblings */ + if (cpumask_subset(&cpu_topology[cpu].core_sibling, core_mask)) { + /* not numa in package, lets use the package siblings */ + core_mask = &cpu_topology[cpu].core_sibling; + } + + if (last_level_cache_is_valid(cpu)) { + if (cpumask_subset(&cpu_topology[cpu].llc_sibling, core_mask)) + core_mask = &cpu_topology[cpu].llc_sibling; + } + + /* + * For systems with no shared cpu-side LLC but with clusters defined, + * extend core_mask to cluster_siblings. The sched domain builder will + * then remove MC as redundant with CLS if SCHED_CLUSTER is enabled. + */ + if (IS_ENABLED(CONFIG_SCHED_CLUSTER) && + cpumask_subset(core_mask, &cpu_topology[cpu].cluster_sibling)) + core_mask = &cpu_topology[cpu].cluster_sibling; + + return core_mask; +} + +const struct cpumask *cpu_clustergroup_mask(int cpu) +{ + /* + * Forbid cpu_clustergroup_mask() to span more or the same CPUs as + * cpu_coregroup_mask(). + */ + if (cpumask_subset(cpu_coregroup_mask(cpu), + &cpu_topology[cpu].cluster_sibling)) + return topology_sibling_cpumask(cpu); + + return &cpu_topology[cpu].cluster_sibling; +} + +void update_siblings_masks(unsigned int cpuid) +{ + struct cpu_topology *cpu_topo, *cpuid_topo = &cpu_topology[cpuid]; + int cpu, ret; + + ret = detect_cache_attributes(cpuid); + if (ret && ret != -ENOENT) + pr_info("Early cacheinfo allocation failed, ret = %d\n", ret); + + /* update core and thread sibling masks */ + for_each_online_cpu(cpu) { + cpu_topo = &cpu_topology[cpu]; + + if (last_level_cache_is_shared(cpu, cpuid)) { + cpumask_set_cpu(cpu, &cpuid_topo->llc_sibling); + cpumask_set_cpu(cpuid, &cpu_topo->llc_sibling); + } + + if (cpuid_topo->package_id != cpu_topo->package_id) + continue; + + cpumask_set_cpu(cpuid, &cpu_topo->core_sibling); + cpumask_set_cpu(cpu, &cpuid_topo->core_sibling); + + if (cpuid_topo->cluster_id != cpu_topo->cluster_id) + continue; + + if (cpuid_topo->cluster_id >= 0) { + cpumask_set_cpu(cpu, &cpuid_topo->cluster_sibling); + cpumask_set_cpu(cpuid, &cpu_topo->cluster_sibling); + } + + if (cpuid_topo->core_id != cpu_topo->core_id) + continue; + + cpumask_set_cpu(cpuid, &cpu_topo->thread_sibling); + cpumask_set_cpu(cpu, &cpuid_topo->thread_sibling); + } +} + +static void clear_cpu_topology(int cpu) +{ + struct cpu_topology *cpu_topo = &cpu_topology[cpu]; + + cpumask_clear(&cpu_topo->llc_sibling); + cpumask_set_cpu(cpu, &cpu_topo->llc_sibling); + + cpumask_clear(&cpu_topo->cluster_sibling); + cpumask_set_cpu(cpu, &cpu_topo->cluster_sibling); + + cpumask_clear(&cpu_topo->core_sibling); + cpumask_set_cpu(cpu, &cpu_topo->core_sibling); + cpumask_clear(&cpu_topo->thread_sibling); + cpumask_set_cpu(cpu, &cpu_topo->thread_sibling); +} + +void __init reset_cpu_topology(void) +{ + unsigned int cpu; + + for_each_possible_cpu(cpu) { + struct cpu_topology *cpu_topo = &cpu_topology[cpu]; + + cpu_topo->thread_id = -1; + cpu_topo->core_id = -1; + cpu_topo->cluster_id = -1; + cpu_topo->package_id = -1; + + clear_cpu_topology(cpu); + } +} + +void remove_cpu_topology(unsigned int cpu) +{ + int sibling; + + for_each_cpu(sibling, topology_core_cpumask(cpu)) + cpumask_clear_cpu(cpu, topology_core_cpumask(sibling)); + for_each_cpu(sibling, topology_sibling_cpumask(cpu)) + cpumask_clear_cpu(cpu, topology_sibling_cpumask(sibling)); + for_each_cpu(sibling, topology_cluster_cpumask(cpu)) + cpumask_clear_cpu(cpu, topology_cluster_cpumask(sibling)); + for_each_cpu(sibling, topology_llc_cpumask(cpu)) + cpumask_clear_cpu(cpu, topology_llc_cpumask(sibling)); + + clear_cpu_topology(cpu); +} + +#if defined(CONFIG_ARM64) || defined(CONFIG_RISCV) +struct cpu_smt_info { + unsigned int thread_num; + int core_id; +}; + +static bool __init acpi_cpu_is_threaded(int cpu) +{ + int is_threaded = acpi_pptt_cpu_is_thread(cpu); + + /* + * if the PPTT doesn't have thread information, check for architecture + * specific fallback if available + */ + if (is_threaded < 0) + is_threaded = arch_cpu_is_threaded(); + + return !!is_threaded; +} + +/* + * Propagate the topology information of the processor_topology_node tree to the + * cpu_topology array. + */ +__weak int __init parse_acpi_topology(void) +{ + unsigned int max_smt_thread_num = 1; + struct cpu_smt_info *entry; + struct xarray hetero_cpu; + unsigned long hetero_id; + int cpu, topology_id; + + if (acpi_disabled) + return 0; + + xa_init(&hetero_cpu); + + for_each_possible_cpu(cpu) { + topology_id = find_acpi_cpu_topology(cpu, 0); + if (topology_id < 0) + return topology_id; + + if (acpi_cpu_is_threaded(cpu)) { + cpu_topology[cpu].thread_id = topology_id; + topology_id = find_acpi_cpu_topology(cpu, 1); + cpu_topology[cpu].core_id = topology_id; + + /* + * In the PPTT, CPUs below a node with the 'identical + * implementation' flag have the same number of threads. + * Count the number of threads for only one CPU (i.e. + * one core_id) among those with the same hetero_id. + * See the comment of find_acpi_cpu_topology_hetero_id() + * for more details. + * + * One entry is created for each node having: + * - the 'identical implementation' flag + * - its parent not having the flag + */ + hetero_id = find_acpi_cpu_topology_hetero_id(cpu); + entry = xa_load(&hetero_cpu, hetero_id); + if (!entry) { + entry = kzalloc(sizeof(*entry), GFP_KERNEL); + WARN_ON_ONCE(!entry); + + if (entry) { + entry->core_id = topology_id; + entry->thread_num = 1; + xa_store(&hetero_cpu, hetero_id, + entry, GFP_KERNEL); + } + } else if (entry->core_id == topology_id) { + entry->thread_num++; + } + } else { + cpu_topology[cpu].thread_id = -1; + cpu_topology[cpu].core_id = topology_id; + } + topology_id = find_acpi_cpu_topology_cluster(cpu); + cpu_topology[cpu].cluster_id = topology_id; + topology_id = find_acpi_cpu_topology_package(cpu); + cpu_topology[cpu].package_id = topology_id; + } + + /* + * This is a short loop since the number of XArray elements is the + * number of heterogeneous CPU clusters. On a homogeneous system + * there's only one entry in the XArray. + */ + xa_for_each(&hetero_cpu, hetero_id, entry) { + max_smt_thread_num = max(max_smt_thread_num, entry->thread_num); + xa_erase(&hetero_cpu, hetero_id); + kfree(entry); + } + + cpu_smt_set_num_threads(max_smt_thread_num, max_smt_thread_num); + xa_destroy(&hetero_cpu); + return 0; +} + +void __init init_cpu_topology(void) +{ + int cpu, ret; + + reset_cpu_topology(); + ret = parse_acpi_topology(); + if (!ret) + ret = of_have_populated_dt() && parse_dt_topology(); + + if (ret) { + /* + * Discard anything that was parsed if we hit an error so we + * don't use partial information. But do not return yet to give + * arch-specific early cache level detection a chance to run. + */ + reset_cpu_topology(); + } + + for_each_possible_cpu(cpu) { + ret = fetch_cache_info(cpu); + if (!ret) + continue; + else if (ret != -ENOENT) + pr_err("Early cacheinfo failed, ret = %d\n", ret); + return; + } +} + +void store_cpu_topology(unsigned int cpuid) +{ + struct cpu_topology *cpuid_topo = &cpu_topology[cpuid]; + + if (cpuid_topo->package_id != -1) + goto topology_populated; + + cpuid_topo->thread_id = -1; + cpuid_topo->core_id = cpuid; + cpuid_topo->package_id = cpu_to_node(cpuid); + + pr_debug("CPU%u: package %d core %d thread %d\n", + cpuid, cpuid_topo->package_id, cpuid_topo->core_id, + cpuid_topo->thread_id); + +topology_populated: + update_siblings_masks(cpuid); +} +#endif diff --git a/drivers/base/attribute_container.c b/drivers/base/attribute_container.c index ecc1929d7f6a..b6f941a6ab69 100644 --- a/drivers/base/attribute_container.c +++ b/drivers/base/attribute_container.c @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /* * attribute_container.c - implementation of a simple container for classes * * Copyright (c) 2005 - James Bottomley <James.Bottomley@steeleye.com> * - * This file is licensed under GPLv2 - * * The basic idea here is to enable a device to be attached to an * aritrary numer of classes without having to allocate storage for them. * Instead, the contained classes select the devices they need to attach @@ -12,7 +11,6 @@ */ #include <linux/attribute_container.h> -#include <linux/init.h> #include <linux/device.h> #include <linux/kernel.h> #include <linux/slab.h> @@ -75,9 +73,9 @@ int attribute_container_register(struct attribute_container *cont) { INIT_LIST_HEAD(&cont->node); - klist_init(&cont->containers,internal_container_klist_get, + klist_init(&cont->containers, internal_container_klist_get, internal_container_klist_put); - + mutex_lock(&attribute_container_mutex); list_add_tail(&cont->node, &attribute_container_list); mutex_unlock(&attribute_container_mutex); @@ -95,6 +93,7 @@ int attribute_container_unregister(struct attribute_container *cont) { int retval = -EBUSY; + mutex_lock(&attribute_container_mutex); spin_lock(&cont->containers.k_lock); if (!list_empty(&cont->containers.k_list)) @@ -105,14 +104,14 @@ attribute_container_unregister(struct attribute_container *cont) spin_unlock(&cont->containers.k_lock); mutex_unlock(&attribute_container_mutex); return retval; - + } EXPORT_SYMBOL_GPL(attribute_container_unregister); /* private function used as class release */ static void attribute_container_release(struct device *classdev) { - struct internal_container *ic + struct internal_container *ic = container_of(classdev, struct internal_container, classdev); struct device *dev = classdev->parent; @@ -185,8 +184,8 @@ attribute_container_add_device(struct device *dev, struct klist_node *n = klist_next(iter); \ n ? container_of(n, typeof(*pos), member) : \ ({ klist_iter_exit(iter) ; NULL; }); \ - }) ) != NULL; ) - + })) != NULL;) + /** * attribute_container_remove_device - make device eligible for removal. @@ -237,68 +236,143 @@ attribute_container_remove_device(struct device *dev, mutex_unlock(&attribute_container_mutex); } +static int +do_attribute_container_device_trigger_safe(struct device *dev, + struct attribute_container *cont, + int (*fn)(struct attribute_container *, + struct device *, struct device *), + int (*undo)(struct attribute_container *, + struct device *, struct device *)) +{ + int ret; + struct internal_container *ic, *failed; + struct klist_iter iter; + + if (attribute_container_no_classdevs(cont)) + return fn(cont, dev, NULL); + + klist_for_each_entry(ic, &cont->containers, node, &iter) { + if (dev == ic->classdev.parent) { + ret = fn(cont, dev, &ic->classdev); + if (ret) { + failed = ic; + klist_iter_exit(&iter); + goto fail; + } + } + } + return 0; + +fail: + if (!undo) + return ret; + + /* Attempt to undo the work partially done. */ + klist_for_each_entry(ic, &cont->containers, node, &iter) { + if (ic == failed) { + klist_iter_exit(&iter); + break; + } + if (dev == ic->classdev.parent) + undo(cont, dev, &ic->classdev); + } + return ret; +} + /** - * attribute_container_device_trigger - execute a trigger for each matching classdev + * attribute_container_device_trigger_safe - execute a trigger for each + * matching classdev or fail all of them. * * @dev: The generic device to run the trigger for - * @fn the function to execute for each classdev. + * @fn: the function to execute for each classdev. + * @undo: A function to undo the work previously done in case of error * - * This funcion is for executing a trigger when you need to know both - * the container and the classdev. If you only care about the - * container, then use attribute_container_trigger() instead. + * This function is a safe version of + * attribute_container_device_trigger. It stops on the first error and + * undo the partial work that has been done, on previous classdev. It + * is guaranteed that either they all succeeded, or none of them + * succeeded. */ -void -attribute_container_device_trigger(struct device *dev, - int (*fn)(struct attribute_container *, - struct device *, - struct device *)) +int +attribute_container_device_trigger_safe(struct device *dev, + int (*fn)(struct attribute_container *, + struct device *, + struct device *), + int (*undo)(struct attribute_container *, + struct device *, + struct device *)) { - struct attribute_container *cont; + struct attribute_container *cont, *failed = NULL; + int ret = 0; mutex_lock(&attribute_container_mutex); + list_for_each_entry(cont, &attribute_container_list, node) { - struct internal_container *ic; - struct klist_iter iter; if (!cont->match(cont, dev)) continue; - if (attribute_container_no_classdevs(cont)) { - fn(cont, dev, NULL); - continue; + ret = do_attribute_container_device_trigger_safe(dev, cont, + fn, undo); + if (ret) { + failed = cont; + break; } + } - klist_for_each_entry(ic, &cont->containers, node, &iter) { - if (dev == ic->classdev.parent) - fn(cont, dev, &ic->classdev); + if (ret && !WARN_ON(!undo)) { + list_for_each_entry(cont, &attribute_container_list, node) { + + if (failed == cont) + break; + + if (!cont->match(cont, dev)) + continue; + + do_attribute_container_device_trigger_safe(dev, cont, + undo, NULL); } } + mutex_unlock(&attribute_container_mutex); + return ret; + } /** - * attribute_container_trigger - trigger a function for each matching container + * attribute_container_device_trigger - execute a trigger for each matching classdev * - * @dev: The generic device to activate the trigger for - * @fn: the function to trigger + * @dev: The generic device to run the trigger for + * @fn: the function to execute for each classdev. * - * This routine triggers a function that only needs to know the - * matching containers (not the classdev) associated with a device. - * It is more lightweight than attribute_container_device_trigger, so - * should be used in preference unless the triggering function - * actually needs to know the classdev. + * This function is for executing a trigger when you need to know both + * the container and the classdev. */ void -attribute_container_trigger(struct device *dev, - int (*fn)(struct attribute_container *, - struct device *)) +attribute_container_device_trigger(struct device *dev, + int (*fn)(struct attribute_container *, + struct device *, + struct device *)) { struct attribute_container *cont; mutex_lock(&attribute_container_mutex); list_for_each_entry(cont, &attribute_container_list, node) { - if (cont->match(cont, dev)) - fn(cont, dev); + struct internal_container *ic; + struct klist_iter iter; + + if (!cont->match(cont, dev)) + continue; + + if (attribute_container_no_classdevs(cont)) { + fn(cont, dev, NULL); + continue; + } + + klist_for_each_entry(ic, &cont->containers, node, &iter) { + if (dev == ic->classdev.parent) + fn(cont, dev, &ic->classdev); + } } mutex_unlock(&attribute_container_mutex); } @@ -350,26 +424,13 @@ int attribute_container_add_class_device(struct device *classdev) { int error = device_add(classdev); + if (error) return error; return attribute_container_add_attrs(classdev); } /** - * attribute_container_add_class_device_adapter - simple adapter for triggers - * - * This function is identical to attribute_container_add_class_device except - * that it is designed to be called from the triggers - */ -int -attribute_container_add_class_device_adapter(struct attribute_container *cont, - struct device *dev, - struct device *classdev) -{ - return attribute_container_add_class_device(classdev); -} - -/** * attribute_container_remove_attrs - remove any attribute files * * @classdev: The class device to remove the files from diff --git a/drivers/base/auxiliary.c b/drivers/base/auxiliary.c new file mode 100644 index 000000000000..04bdbff4dbe5 --- /dev/null +++ b/drivers/base/auxiliary.c @@ -0,0 +1,508 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2019-2020 Intel Corporation + * + * Please see Documentation/driver-api/auxiliary_bus.rst for more information. + */ + +#define pr_fmt(fmt) "%s:%s: " fmt, KBUILD_MODNAME, __func__ + +#include <linux/device.h> +#include <linux/init.h> +#include <linux/slab.h> +#include <linux/module.h> +#include <linux/pm_domain.h> +#include <linux/pm_runtime.h> +#include <linux/string.h> +#include <linux/auxiliary_bus.h> +#include "base.h" + +/** + * DOC: PURPOSE + * + * In some subsystems, the functionality of the core device (PCI/ACPI/other) is + * too complex for a single device to be managed by a monolithic driver (e.g. + * Sound Open Firmware), multiple devices might implement a common intersection + * of functionality (e.g. NICs + RDMA), or a driver may want to export an + * interface for another subsystem to drive (e.g. SIOV Physical Function export + * Virtual Function management). A split of the functionality into child- + * devices representing sub-domains of functionality makes it possible to + * compartmentalize, layer, and distribute domain-specific concerns via a Linux + * device-driver model. + * + * An example for this kind of requirement is the audio subsystem where a + * single IP is handling multiple entities such as HDMI, Soundwire, local + * devices such as mics/speakers etc. The split for the core's functionality + * can be arbitrary or be defined by the DSP firmware topology and include + * hooks for test/debug. This allows for the audio core device to be minimal + * and focused on hardware-specific control and communication. + * + * Each auxiliary_device represents a part of its parent functionality. The + * generic behavior can be extended and specialized as needed by encapsulating + * an auxiliary_device within other domain-specific structures and the use of + * .ops callbacks. Devices on the auxiliary bus do not share any structures and + * the use of a communication channel with the parent is domain-specific. + * + * Note that ops are intended as a way to augment instance behavior within a + * class of auxiliary devices, it is not the mechanism for exporting common + * infrastructure from the parent. Consider EXPORT_SYMBOL_NS() to convey + * infrastructure from the parent module to the auxiliary module(s). + */ + +/** + * DOC: USAGE + * + * The auxiliary bus is to be used when a driver and one or more kernel + * modules, who share a common header file with the driver, need a mechanism to + * connect and provide access to a shared object allocated by the + * auxiliary_device's registering driver. The registering driver for the + * auxiliary_device(s) and the kernel module(s) registering auxiliary_drivers + * can be from the same subsystem, or from multiple subsystems. + * + * The emphasis here is on a common generic interface that keeps subsystem + * customization out of the bus infrastructure. + * + * One example is a PCI network device that is RDMA-capable and exports a child + * device to be driven by an auxiliary_driver in the RDMA subsystem. The PCI + * driver allocates and registers an auxiliary_device for each physical + * function on the NIC. The RDMA driver registers an auxiliary_driver that + * claims each of these auxiliary_devices. This conveys data/ops published by + * the parent PCI device/driver to the RDMA auxiliary_driver. + * + * Another use case is for the PCI device to be split out into multiple sub + * functions. For each sub function an auxiliary_device is created. A PCI sub + * function driver binds to such devices that creates its own one or more class + * devices. A PCI sub function auxiliary device is likely to be contained in a + * struct with additional attributes such as user defined sub function number + * and optional attributes such as resources and a link to the parent device. + * These attributes could be used by systemd/udev; and hence should be + * initialized before a driver binds to an auxiliary_device. + * + * A key requirement for utilizing the auxiliary bus is that there is no + * dependency on a physical bus, device, register accesses or regmap support. + * These individual devices split from the core cannot live on the platform bus + * as they are not physical devices that are controlled by DT/ACPI. The same + * argument applies for not using MFD in this scenario as MFD relies on + * individual function devices being physical devices. + */ + +/** + * DOC: EXAMPLE + * + * Auxiliary devices are created and registered by a subsystem-level core + * device that needs to break up its functionality into smaller fragments. One + * way to extend the scope of an auxiliary_device is to encapsulate it within a + * domain-specific structure defined by the parent device. This structure + * contains the auxiliary_device and any associated shared data/callbacks + * needed to establish the connection with the parent. + * + * An example is: + * + * .. code-block:: c + * + * struct foo { + * struct auxiliary_device auxdev; + * void (*connect)(struct auxiliary_device *auxdev); + * void (*disconnect)(struct auxiliary_device *auxdev); + * void *data; + * }; + * + * The parent device then registers the auxiliary_device by calling + * auxiliary_device_init(), and then auxiliary_device_add(), with the pointer + * to the auxdev member of the above structure. The parent provides a name for + * the auxiliary_device that, combined with the parent's KBUILD_MODNAME, + * creates a match_name that is be used for matching and binding with a driver. + * + * Whenever an auxiliary_driver is registered, based on the match_name, the + * auxiliary_driver's probe() is invoked for the matching devices. The + * auxiliary_driver can also be encapsulated inside custom drivers that make + * the core device's functionality extensible by adding additional + * domain-specific ops as follows: + * + * .. code-block:: c + * + * struct my_ops { + * void (*send)(struct auxiliary_device *auxdev); + * void (*receive)(struct auxiliary_device *auxdev); + * }; + * + * + * struct my_driver { + * struct auxiliary_driver auxiliary_drv; + * const struct my_ops ops; + * }; + * + * An example of this type of usage is: + * + * .. code-block:: c + * + * const struct auxiliary_device_id my_auxiliary_id_table[] = { + * { .name = "foo_mod.foo_dev" }, + * { }, + * }; + * + * const struct my_ops my_custom_ops = { + * .send = my_tx, + * .receive = my_rx, + * }; + * + * const struct my_driver my_drv = { + * .auxiliary_drv = { + * .name = "myauxiliarydrv", + * .id_table = my_auxiliary_id_table, + * .probe = my_probe, + * .remove = my_remove, + * .shutdown = my_shutdown, + * }, + * .ops = my_custom_ops, + * }; + * + * Please note that such custom ops approach is valid, but it is hard to implement + * it right without global locks per-device to protect from auxiliary_drv removal + * during call to that ops. In addition, this implementation lacks proper module + * dependency, which causes to load/unload races between auxiliary parent and devices + * modules. + * + * The most easiest way to provide these ops reliably without needing to + * have a lock is to EXPORT_SYMBOL*() them and rely on already existing + * modules infrastructure for validity and correct dependencies chains. + */ + +static const struct auxiliary_device_id *auxiliary_match_id(const struct auxiliary_device_id *id, + const struct auxiliary_device *auxdev) +{ + const char *auxdev_name = dev_name(&auxdev->dev); + const char *p = strrchr(auxdev_name, '.'); + int match_size; + + if (!p) + return NULL; + match_size = p - auxdev_name; + + for (; id->name[0]; id++) { + /* use dev_name(&auxdev->dev) prefix before last '.' char to match to */ + if (strlen(id->name) == match_size && + !strncmp(auxdev_name, id->name, match_size)) + return id; + } + return NULL; +} + +static int auxiliary_match(struct device *dev, const struct device_driver *drv) +{ + struct auxiliary_device *auxdev = to_auxiliary_dev(dev); + const struct auxiliary_driver *auxdrv = to_auxiliary_drv(drv); + + return !!auxiliary_match_id(auxdrv->id_table, auxdev); +} + +static int auxiliary_uevent(const struct device *dev, struct kobj_uevent_env *env) +{ + const char *name, *p; + + name = dev_name(dev); + p = strrchr(name, '.'); + + return add_uevent_var(env, "MODALIAS=%s%.*s", AUXILIARY_MODULE_PREFIX, + (int)(p - name), name); +} + +static const struct dev_pm_ops auxiliary_dev_pm_ops = { + SET_RUNTIME_PM_OPS(pm_generic_runtime_suspend, pm_generic_runtime_resume, NULL) + SET_SYSTEM_SLEEP_PM_OPS(pm_generic_suspend, pm_generic_resume) +}; + +static int auxiliary_bus_probe(struct device *dev) +{ + const struct auxiliary_driver *auxdrv = to_auxiliary_drv(dev->driver); + struct auxiliary_device *auxdev = to_auxiliary_dev(dev); + int ret; + + ret = dev_pm_domain_attach(dev, PD_FLAG_ATTACH_POWER_ON | + PD_FLAG_DETACH_POWER_OFF); + if (ret) { + dev_warn(dev, "Failed to attach to PM Domain : %d\n", ret); + return ret; + } + + return auxdrv->probe(auxdev, auxiliary_match_id(auxdrv->id_table, auxdev)); +} + +static void auxiliary_bus_remove(struct device *dev) +{ + const struct auxiliary_driver *auxdrv = to_auxiliary_drv(dev->driver); + struct auxiliary_device *auxdev = to_auxiliary_dev(dev); + + if (auxdrv->remove) + auxdrv->remove(auxdev); +} + +static void auxiliary_bus_shutdown(struct device *dev) +{ + const struct auxiliary_driver *auxdrv = NULL; + struct auxiliary_device *auxdev; + + if (dev->driver) { + auxdrv = to_auxiliary_drv(dev->driver); + auxdev = to_auxiliary_dev(dev); + } + + if (auxdrv && auxdrv->shutdown) + auxdrv->shutdown(auxdev); +} + +static const struct bus_type auxiliary_bus_type = { + .name = "auxiliary", + .probe = auxiliary_bus_probe, + .remove = auxiliary_bus_remove, + .shutdown = auxiliary_bus_shutdown, + .match = auxiliary_match, + .uevent = auxiliary_uevent, + .pm = &auxiliary_dev_pm_ops, +}; + +/** + * auxiliary_device_init - check auxiliary_device and initialize + * @auxdev: auxiliary device struct + * + * This is the second step in the three-step process to register an + * auxiliary_device. + * + * When this function returns an error code, then the device_initialize will + * *not* have been performed, and the caller will be responsible to free any + * memory allocated for the auxiliary_device in the error path directly. + * + * It returns 0 on success. On success, the device_initialize has been + * performed. After this point any error unwinding will need to include a call + * to auxiliary_device_uninit(). In this post-initialize error scenario, a call + * to the device's .release callback will be triggered, and all memory clean-up + * is expected to be handled there. + */ +int auxiliary_device_init(struct auxiliary_device *auxdev) +{ + struct device *dev = &auxdev->dev; + + if (!dev->parent) { + pr_err("auxiliary_device has a NULL dev->parent\n"); + return -EINVAL; + } + + if (!auxdev->name) { + pr_err("auxiliary_device has a NULL name\n"); + return -EINVAL; + } + + dev->bus = &auxiliary_bus_type; + device_initialize(&auxdev->dev); + mutex_init(&auxdev->sysfs.lock); + return 0; +} +EXPORT_SYMBOL_GPL(auxiliary_device_init); + +/** + * __auxiliary_device_add - add an auxiliary bus device + * @auxdev: auxiliary bus device to add to the bus + * @modname: name of the parent device's driver module + * + * This is the third step in the three-step process to register an + * auxiliary_device. + * + * This function must be called after a successful call to + * auxiliary_device_init(), which will perform the device_initialize. This + * means that if this returns an error code, then a call to + * auxiliary_device_uninit() must be performed so that the .release callback + * will be triggered to free the memory associated with the auxiliary_device. + * + * The expectation is that users will call the "auxiliary_device_add" macro so + * that the caller's KBUILD_MODNAME is automatically inserted for the modname + * parameter. Only if a user requires a custom name would this version be + * called directly. + */ +int __auxiliary_device_add(struct auxiliary_device *auxdev, const char *modname) +{ + struct device *dev = &auxdev->dev; + int ret; + + if (!modname) { + dev_err(dev, "auxiliary device modname is NULL\n"); + return -EINVAL; + } + + ret = dev_set_name(dev, "%s.%s.%d", modname, auxdev->name, auxdev->id); + if (ret) { + dev_err(dev, "auxiliary device dev_set_name failed: %d\n", ret); + return ret; + } + + ret = device_add(dev); + if (ret) + dev_err(dev, "adding auxiliary device failed!: %d\n", ret); + + return ret; +} +EXPORT_SYMBOL_GPL(__auxiliary_device_add); + +/** + * __auxiliary_driver_register - register a driver for auxiliary bus devices + * @auxdrv: auxiliary_driver structure + * @owner: owning module/driver + * @modname: KBUILD_MODNAME for parent driver + * + * The expectation is that users will call the "auxiliary_driver_register" + * macro so that the caller's KBUILD_MODNAME is automatically inserted for the + * modname parameter. Only if a user requires a custom name would this version + * be called directly. + */ +int __auxiliary_driver_register(struct auxiliary_driver *auxdrv, + struct module *owner, const char *modname) +{ + int ret; + + if (WARN_ON(!auxdrv->probe) || WARN_ON(!auxdrv->id_table)) + return -EINVAL; + + if (auxdrv->name) + auxdrv->driver.name = kasprintf(GFP_KERNEL, "%s.%s", modname, + auxdrv->name); + else + auxdrv->driver.name = kasprintf(GFP_KERNEL, "%s", modname); + if (!auxdrv->driver.name) + return -ENOMEM; + + auxdrv->driver.owner = owner; + auxdrv->driver.bus = &auxiliary_bus_type; + auxdrv->driver.mod_name = modname; + + ret = driver_register(&auxdrv->driver); + if (ret) + kfree(auxdrv->driver.name); + + return ret; +} +EXPORT_SYMBOL_GPL(__auxiliary_driver_register); + +/** + * auxiliary_driver_unregister - unregister a driver + * @auxdrv: auxiliary_driver structure + */ +void auxiliary_driver_unregister(struct auxiliary_driver *auxdrv) +{ + driver_unregister(&auxdrv->driver); + kfree(auxdrv->driver.name); +} +EXPORT_SYMBOL_GPL(auxiliary_driver_unregister); + +static void auxiliary_device_release(struct device *dev) +{ + struct auxiliary_device *auxdev = to_auxiliary_dev(dev); + + of_node_put(dev->of_node); + kfree(auxdev); +} + +/** + * auxiliary_device_create - create a device on the auxiliary bus + * @dev: parent device + * @modname: module name used to create the auxiliary driver name. + * @devname: auxiliary bus device name + * @platform_data: auxiliary bus device platform data + * @id: auxiliary bus device id + * + * Helper to create an auxiliary bus device. + * The device created matches driver 'modname.devname' on the auxiliary bus. + */ +struct auxiliary_device *auxiliary_device_create(struct device *dev, + const char *modname, + const char *devname, + void *platform_data, + int id) +{ + struct auxiliary_device *auxdev; + int ret; + + auxdev = kzalloc(sizeof(*auxdev), GFP_KERNEL); + if (!auxdev) + return NULL; + + auxdev->id = id; + auxdev->name = devname; + auxdev->dev.parent = dev; + auxdev->dev.platform_data = platform_data; + auxdev->dev.release = auxiliary_device_release; + device_set_of_node_from_dev(&auxdev->dev, dev); + + ret = auxiliary_device_init(auxdev); + if (ret) { + of_node_put(auxdev->dev.of_node); + kfree(auxdev); + return NULL; + } + + ret = __auxiliary_device_add(auxdev, modname); + if (ret) { + /* + * It may look odd but auxdev should not be freed here. + * auxiliary_device_uninit() calls device_put() which call + * the device release function, freeing auxdev. + */ + auxiliary_device_uninit(auxdev); + return NULL; + } + + return auxdev; +} +EXPORT_SYMBOL_GPL(auxiliary_device_create); + +/** + * auxiliary_device_destroy - remove an auxiliary device + * @auxdev: pointer to the auxdev to be removed + * + * Helper to remove an auxiliary device created with + * auxiliary_device_create() + */ +void auxiliary_device_destroy(void *auxdev) +{ + struct auxiliary_device *_auxdev = auxdev; + + auxiliary_device_delete(_auxdev); + auxiliary_device_uninit(_auxdev); +} +EXPORT_SYMBOL_GPL(auxiliary_device_destroy); + +/** + * __devm_auxiliary_device_create - create a managed device on the auxiliary bus + * @dev: parent device + * @modname: module name used to create the auxiliary driver name. + * @devname: auxiliary bus device name + * @platform_data: auxiliary bus device platform data + * @id: auxiliary bus device id + * + * Device managed helper to create an auxiliary bus device. + * The device created matches driver 'modname.devname' on the auxiliary bus. + */ +struct auxiliary_device *__devm_auxiliary_device_create(struct device *dev, + const char *modname, + const char *devname, + void *platform_data, + int id) +{ + struct auxiliary_device *auxdev; + int ret; + + auxdev = auxiliary_device_create(dev, modname, devname, platform_data, id); + if (!auxdev) + return NULL; + + ret = devm_add_action_or_reset(dev, auxiliary_device_destroy, + auxdev); + if (ret) + return NULL; + + return auxdev; +} +EXPORT_SYMBOL_GPL(__devm_auxiliary_device_create); + +void __init auxiliary_bus_init(void) +{ + WARN_ON(bus_register(&auxiliary_bus_type)); +} diff --git a/drivers/base/auxiliary_sysfs.c b/drivers/base/auxiliary_sysfs.c new file mode 100644 index 000000000000..754f21730afd --- /dev/null +++ b/drivers/base/auxiliary_sysfs.c @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES + */ + +#include <linux/auxiliary_bus.h> +#include <linux/slab.h> + +#define AUXILIARY_MAX_IRQ_NAME 11 + +struct auxiliary_irq_info { + struct device_attribute sysfs_attr; + char name[AUXILIARY_MAX_IRQ_NAME]; +}; + +static struct attribute *auxiliary_irq_attrs[] = { + NULL +}; + +static const struct attribute_group auxiliary_irqs_group = { + .name = "irqs", + .attrs = auxiliary_irq_attrs, +}; + +static int auxiliary_irq_dir_prepare(struct auxiliary_device *auxdev) +{ + int ret = 0; + + guard(mutex)(&auxdev->sysfs.lock); + if (auxdev->sysfs.irq_dir_exists) + return 0; + + ret = devm_device_add_group(&auxdev->dev, &auxiliary_irqs_group); + if (ret) + return ret; + + auxdev->sysfs.irq_dir_exists = true; + xa_init(&auxdev->sysfs.irqs); + return 0; +} + +/** + * auxiliary_device_sysfs_irq_add - add a sysfs entry for the given IRQ + * @auxdev: auxiliary bus device to add the sysfs entry. + * @irq: The associated interrupt number. + * + * This function should be called after auxiliary device have successfully + * received the irq. + * The driver is responsible to add a unique irq for the auxiliary device. The + * driver can invoke this function from multiple thread context safely for + * unique irqs of the auxiliary devices. The driver must not invoke this API + * multiple times if the irq is already added previously. + * + * Return: zero on success or an error code on failure. + */ +int auxiliary_device_sysfs_irq_add(struct auxiliary_device *auxdev, int irq) +{ + struct auxiliary_irq_info *info __free(kfree) = NULL; + struct device *dev = &auxdev->dev; + int ret; + + ret = auxiliary_irq_dir_prepare(auxdev); + if (ret) + return ret; + + info = kzalloc(sizeof(*info), GFP_KERNEL); + if (!info) + return -ENOMEM; + + sysfs_attr_init(&info->sysfs_attr.attr); + snprintf(info->name, AUXILIARY_MAX_IRQ_NAME, "%d", irq); + + ret = xa_insert(&auxdev->sysfs.irqs, irq, info, GFP_KERNEL); + if (ret) + return ret; + + info->sysfs_attr.attr.name = info->name; + ret = sysfs_add_file_to_group(&dev->kobj, &info->sysfs_attr.attr, + auxiliary_irqs_group.name); + if (ret) + goto sysfs_add_err; + + xa_store(&auxdev->sysfs.irqs, irq, no_free_ptr(info), GFP_KERNEL); + return 0; + +sysfs_add_err: + xa_erase(&auxdev->sysfs.irqs, irq); + return ret; +} +EXPORT_SYMBOL_GPL(auxiliary_device_sysfs_irq_add); + +/** + * auxiliary_device_sysfs_irq_remove - remove a sysfs entry for the given IRQ + * @auxdev: auxiliary bus device to add the sysfs entry. + * @irq: the IRQ to remove. + * + * This function should be called to remove an IRQ sysfs entry. + * The driver must invoke this API when IRQ is released by the device. + */ +void auxiliary_device_sysfs_irq_remove(struct auxiliary_device *auxdev, int irq) +{ + struct auxiliary_irq_info *info __free(kfree) = xa_load(&auxdev->sysfs.irqs, irq); + struct device *dev = &auxdev->dev; + + if (!info) { + dev_err(&auxdev->dev, "IRQ %d doesn't exist\n", irq); + return; + } + sysfs_remove_file_from_group(&dev->kobj, &info->sysfs_attr.attr, + auxiliary_irqs_group.name); + xa_erase(&auxdev->sysfs.irqs, irq); +} +EXPORT_SYMBOL_GPL(auxiliary_device_sysfs_irq_remove); diff --git a/drivers/base/base.h b/drivers/base/base.h index b8bdfe61daa6..430cbefbc97f 100644 --- a/drivers/base/base.h +++ b/drivers/base/base.h @@ -1,3 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2001-2003 Patrick Mochel <mochel@osdl.org> + * Copyright (c) 2004-2009 Greg Kroah-Hartman <gregkh@suse.de> + * Copyright (c) 2008-2012 Novell Inc. + * Copyright (c) 2012-2019 Greg Kroah-Hartman <gregkh@linuxfoundation.org> + * Copyright (c) 2012-2019 Linux Foundation + * + * Core driver model functions and structures that should not be + * shared outside of the drivers/base/ directory. + * + */ #include <linux/notifier.h> /** @@ -15,11 +27,13 @@ * on this bus. * @bus - pointer back to the struct bus_type that this structure is associated * with. + * @dev_root: Default device to use as the parent. * * @glue_dirs - "glue" directory to put in-between the parent device to * avoid namespace conflicts * @class - pointer back to the struct class that this structure is associated * with. + * @lock_key: Lock class key for use by the lock validator * * This structure is the one that is the actual kobject allowing struct * bus_type/class to be statically allocated safely. Nothing outside of the @@ -36,12 +50,31 @@ struct subsys_private { struct klist klist_drivers; struct blocking_notifier_head bus_notifier; unsigned int drivers_autoprobe:1; - struct bus_type *bus; + const struct bus_type *bus; + struct device *dev_root; struct kset glue_dirs; - struct class *class; + const struct class *class; + + struct lock_class_key lock_key; }; -#define to_subsys_private(obj) container_of(obj, struct subsys_private, subsys.kobj) +#define to_subsys_private(obj) container_of_const(obj, struct subsys_private, subsys.kobj) + +static inline struct subsys_private *subsys_get(struct subsys_private *sp) +{ + if (sp) + kset_get(&sp->subsys); + return sp; +} + +static inline void subsys_put(struct subsys_private *sp) +{ + if (sp) + kset_put(&sp->subsys); +} + +struct subsys_private *bus_to_subsys(const struct bus_type *bus); +struct subsys_private *class_to_subsys(const struct class *class); struct driver_private { struct kobject kobj; @@ -52,6 +85,18 @@ struct driver_private { }; #define to_driver(obj) container_of(obj, struct driver_private, kobj) +#ifdef CONFIG_RUST +/** + * struct driver_type - Representation of a Rust driver type. + */ +struct driver_type { + /** + * @id: Representation of core::any::TypeId. + */ + u8 id[16]; +} __packed; +#endif + /** * struct device_private - structure to hold the private to the driver core portions of the device structure. * @@ -59,14 +104,18 @@ struct driver_private { * @knode_parent - node in sibling list * @knode_driver - node in driver list * @knode_bus - node in bus list + * @knode_class - node in class list * @deferred_probe - entry in deferred_probe_list which is used to retry the * binding of drivers which were unable to get all the resources needed by * the device; typically because it depends on another driver getting * probed first. - * @driver_data - private pointer for driver specific info. Will turn into a - * list soon. - * @device - pointer back to the struct class that this structure is + * @async_driver - pointer to device driver awaiting probe via async_probe + * @device - pointer back to the struct device that this structure is * associated with. + * @driver_type - The type of the bound Rust driver. + * @dead - This device is currently either in the process of or has been + * removed from the system. Any asynchronous events scheduled for this + * device should exit without taking any action. * * Nothing outside of the driver core should ever touch these fields. */ @@ -75,9 +124,15 @@ struct device_private { struct klist_node knode_parent; struct klist_node knode_driver; struct klist_node knode_bus; + struct klist_node knode_class; struct list_head deferred_probe; - void *driver_data; + const struct device_driver *async_driver; + char *deferred_probe_reason; struct device *device; +#ifdef CONFIG_RUST + struct driver_type driver_type; +#endif + u8 dead:1; }; #define to_device_private_parent(obj) \ container_of(obj, struct device_private, knode_parent) @@ -85,58 +140,152 @@ struct device_private { container_of(obj, struct device_private, knode_driver) #define to_device_private_bus(obj) \ container_of(obj, struct device_private, knode_bus) - -extern int device_private_init(struct device *dev); +#define to_device_private_class(obj) \ + container_of(obj, struct device_private, knode_class) /* initialisation functions */ -extern int devices_init(void); -extern int buses_init(void); -extern int classes_init(void); -extern int firmware_init(void); +int devices_init(void); +int buses_init(void); +int classes_init(void); +int firmware_init(void); #ifdef CONFIG_SYS_HYPERVISOR -extern int hypervisor_init(void); +int hypervisor_init(void); #else static inline int hypervisor_init(void) { return 0; } #endif -extern int platform_bus_init(void); -extern void cpu_dev_init(void); +int platform_bus_init(void); +int faux_bus_init(void); +void cpu_dev_init(void); +void container_dev_init(void); +#ifdef CONFIG_AUXILIARY_BUS +void auxiliary_bus_init(void); +#else +static inline void auxiliary_bus_init(void) { } +#endif -struct kobject *virtual_device_parent(struct device *dev); +struct kobject *virtual_device_parent(void); -extern int bus_add_device(struct device *dev); -extern void bus_probe_device(struct device *dev); -extern void bus_remove_device(struct device *dev); +int bus_add_device(struct device *dev); +void bus_probe_device(struct device *dev); +void bus_remove_device(struct device *dev); +void bus_notify(struct device *dev, enum bus_notifier_event value); +bool bus_is_registered(const struct bus_type *bus); -extern int bus_add_driver(struct device_driver *drv); -extern void bus_remove_driver(struct device_driver *drv); +int bus_add_driver(struct device_driver *drv); +void bus_remove_driver(struct device_driver *drv); +void device_release_driver_internal(struct device *dev, const struct device_driver *drv, + struct device *parent); -extern void driver_detach(struct device_driver *drv); -extern int driver_probe_device(struct device_driver *drv, struct device *dev); -extern void driver_deferred_probe_del(struct device *dev); -static inline int driver_match_device(struct device_driver *drv, +void driver_detach(const struct device_driver *drv); +void driver_deferred_probe_del(struct device *dev); +void device_set_deferred_probe_reason(const struct device *dev, struct va_format *vaf); +static inline int driver_match_device(const struct device_driver *drv, struct device *dev) { return drv->bus->match ? drv->bus->match(dev, drv) : 1; } -extern char *make_class_name(const char *name, struct kobject *kobj); +static inline void dev_sync_state(struct device *dev) +{ + if (dev->bus->sync_state) + dev->bus->sync_state(dev); + else if (dev->driver && dev->driver->sync_state) + dev->driver->sync_state(dev); +} -extern int devres_release_all(struct device *dev); +int driver_add_groups(const struct device_driver *drv, const struct attribute_group **groups); +void driver_remove_groups(const struct device_driver *drv, const struct attribute_group **groups); +void device_driver_detach(struct device *dev); + +static inline void device_set_driver(struct device *dev, const struct device_driver *drv) +{ + /* + * Majority (all?) read accesses to dev->driver happens either + * while holding device lock or in bus/driver code that is only + * invoked when the device is bound to a driver and there is no + * concern of the pointer being changed while it is being read. + * However when reading device's uevent file we read driver pointer + * without taking device lock (so we do not block there for + * arbitrary amount of time). We use WRITE_ONCE() here to prevent + * tearing so that READ_ONCE() can safely be used in uevent code. + */ + // FIXME - this cast should not be needed "soon" + WRITE_ONCE(dev->driver, (struct device_driver *)drv); +} + +int devres_release_all(struct device *dev); +void device_block_probing(void); +void device_unblock_probing(void); +void deferred_probe_extend_timeout(void); +void driver_deferred_probe_trigger(void); +const char *device_get_devnode(const struct device *dev, umode_t *mode, + kuid_t *uid, kgid_t *gid, const char **tmp); /* /sys/devices directory */ extern struct kset *devices_kset; +void devices_kset_move_last(struct device *dev); #if defined(CONFIG_MODULES) && defined(CONFIG_SYSFS) -extern void module_add_driver(struct module *mod, struct device_driver *drv); -extern void module_remove_driver(struct device_driver *drv); +int module_add_driver(struct module *mod, const struct device_driver *drv); +void module_remove_driver(const struct device_driver *drv); #else -static inline void module_add_driver(struct module *mod, - struct device_driver *drv) { } +static inline int module_add_driver(struct module *mod, + struct device_driver *drv) +{ + return 0; +} static inline void module_remove_driver(struct device_driver *drv) { } #endif #ifdef CONFIG_DEVTMPFS -extern int devtmpfs_init(void); +int devtmpfs_init(void); #else static inline int devtmpfs_init(void) { return 0; } #endif + +#ifdef CONFIG_BLOCK +extern const struct class block_class; +static inline bool is_blockdev(struct device *dev) +{ + return dev->class == &block_class; +} +#else +static inline bool is_blockdev(struct device *dev) { return false; } +#endif + +/* Device links support */ +int device_links_read_lock(void); +void device_links_read_unlock(int idx); +int device_links_read_lock_held(void); +int device_links_check_suppliers(struct device *dev); +void device_links_force_bind(struct device *dev); +void device_links_driver_bound(struct device *dev); +void device_links_driver_cleanup(struct device *dev); +void device_links_no_driver(struct device *dev); +bool device_links_busy(struct device *dev); +void device_links_unbind_consumers(struct device *dev); +bool device_link_flag_is_sync_state_only(u32 flags); +void fw_devlink_drivers_done(void); +void fw_devlink_probing_done(void); + +#define dev_for_each_link_to_supplier(__link, __dev) \ + list_for_each_entry_srcu(__link, &(__dev)->links.suppliers, c_node, \ + device_links_read_lock_held()) + +#define dev_for_each_link_to_consumer(__link, __dev) \ + list_for_each_entry_srcu(__link, &(__dev)->links.consumers, s_node, \ + device_links_read_lock_held()) + +/* device pm support */ +void device_pm_move_to_tail(struct device *dev); + +#ifdef CONFIG_DEVTMPFS +int devtmpfs_create_node(struct device *dev); +int devtmpfs_delete_node(struct device *dev); +#else +static inline int devtmpfs_create_node(struct device *dev) { return 0; } +static inline int devtmpfs_delete_node(struct device *dev) { return 0; } +#endif + +void software_node_notify(struct device *dev); +void software_node_notify_remove(struct device *dev); diff --git a/drivers/base/bus.c b/drivers/base/bus.c index d414331b480e..9eb7771706f0 100644 --- a/drivers/base/bus.c +++ b/drivers/base/bus.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * bus.c - bus driver management * @@ -5,11 +6,11 @@ * Copyright (c) 2002-3 Open Source Development Labs * Copyright (c) 2007 Greg Kroah-Hartman <gregkh@suse.de> * Copyright (c) 2007 Novell Inc. - * - * This file is released under the GPLv2 - * + * Copyright (c) 2023 Greg Kroah-Hartman <gregkh@linuxfoundation.org> */ +#include <linux/async.h> +#include <linux/device/bus.h> #include <linux/device.h> #include <linux/module.h> #include <linux/errno.h> @@ -17,12 +18,16 @@ #include <linux/init.h> #include <linux/string.h> #include <linux/mutex.h> +#include <linux/sysfs.h> #include "base.h" #include "power/power.h" /* /sys/devices/system */ static struct kset *system_kset; +/* /sys/bus */ +static struct kset *bus_kset; + #define to_bus_attr(_attr) container_of(_attr, struct bus_attribute, attr) /* @@ -31,23 +36,70 @@ static struct kset *system_kset; #define to_drv_attr(_attr) container_of(_attr, struct driver_attribute, attr) +#define DRIVER_ATTR_IGNORE_LOCKDEP(_name, _mode, _show, _store) \ + struct driver_attribute driver_attr_##_name = \ + __ATTR_IGNORE_LOCKDEP(_name, _mode, _show, _store) static int __must_check bus_rescan_devices_helper(struct device *dev, void *data); -static struct bus_type *bus_get(struct bus_type *bus) +/** + * bus_to_subsys - Turn a struct bus_type into a struct subsys_private + * + * @bus: pointer to the struct bus_type to look up + * + * The driver core internals needs to work on the subsys_private structure, not + * the external struct bus_type pointer. This function walks the list of + * registered busses in the system and finds the matching one and returns the + * internal struct subsys_private that relates to that bus. + * + * Note, the reference count of the return value is INCREMENTED if it is not + * NULL. A call to subsys_put() must be done when finished with the pointer in + * order for it to be properly freed. + */ +struct subsys_private *bus_to_subsys(const struct bus_type *bus) { - if (bus) { - kset_get(&bus->p->subsys); - return bus; + struct subsys_private *sp = NULL; + struct kobject *kobj; + + if (!bus || !bus_kset) + return NULL; + + spin_lock(&bus_kset->list_lock); + + if (list_empty(&bus_kset->list)) + goto done; + + list_for_each_entry(kobj, &bus_kset->list, entry) { + struct kset *kset = container_of(kobj, struct kset, kobj); + + sp = container_of_const(kset, struct subsys_private, subsys); + if (sp->bus == bus) + goto done; } + sp = NULL; +done: + sp = subsys_get(sp); + spin_unlock(&bus_kset->list_lock); + return sp; +} + +static const struct bus_type *bus_get(const struct bus_type *bus) +{ + struct subsys_private *sp = bus_to_subsys(bus); + + if (sp) + return bus; return NULL; } -static void bus_put(struct bus_type *bus) +static void bus_put(const struct bus_type *bus) { - if (bus) - kset_put(&bus->p->subsys); + struct subsys_private *sp = bus_to_subsys(bus); + + /* two puts are required as the call to bus_to_subsys incremented it again */ + subsys_put(sp); + subsys_put(sp); } static ssize_t drv_attr_show(struct kobject *kobj, struct attribute *attr, @@ -87,7 +139,7 @@ static void driver_release(struct kobject *kobj) kfree(drv_priv); } -static struct kobj_type driver_ktype = { +static const struct kobj_type driver_ktype = { .sysfs_ops = &driver_sysfs_ops, .release = driver_release, }; @@ -100,7 +152,8 @@ static ssize_t bus_attr_show(struct kobject *kobj, struct attribute *attr, { struct bus_attribute *bus_attr = to_bus_attr(attr); struct subsys_private *subsys_priv = to_subsys_private(kobj); - ssize_t ret = 0; + /* return -EIO for reading a bus attribute without show() */ + ssize_t ret = -EIO; if (bus_attr->show) ret = bus_attr->show(subsys_priv->bus, buf); @@ -112,7 +165,8 @@ static ssize_t bus_attr_store(struct kobject *kobj, struct attribute *attr, { struct bus_attribute *bus_attr = to_bus_attr(attr); struct subsys_private *subsys_priv = to_subsys_private(kobj); - ssize_t ret = 0; + /* return -EIO for writing a bus attribute without store() */ + ssize_t ret = -EIO; if (bus_attr->store) ret = bus_attr->store(subsys_priv->bus, buf, count); @@ -124,34 +178,49 @@ static const struct sysfs_ops bus_sysfs_ops = { .store = bus_attr_store, }; -int bus_create_file(struct bus_type *bus, struct bus_attribute *attr) +int bus_create_file(const struct bus_type *bus, struct bus_attribute *attr) { + struct subsys_private *sp = bus_to_subsys(bus); int error; - if (bus_get(bus)) { - error = sysfs_create_file(&bus->p->subsys.kobj, &attr->attr); - bus_put(bus); - } else - error = -EINVAL; + + if (!sp) + return -EINVAL; + + error = sysfs_create_file(&sp->subsys.kobj, &attr->attr); + + subsys_put(sp); return error; } EXPORT_SYMBOL_GPL(bus_create_file); -void bus_remove_file(struct bus_type *bus, struct bus_attribute *attr) +void bus_remove_file(const struct bus_type *bus, struct bus_attribute *attr) { - if (bus_get(bus)) { - sysfs_remove_file(&bus->p->subsys.kobj, &attr->attr); - bus_put(bus); - } + struct subsys_private *sp = bus_to_subsys(bus); + + if (!sp) + return; + + sysfs_remove_file(&sp->subsys.kobj, &attr->attr); + subsys_put(sp); } EXPORT_SYMBOL_GPL(bus_remove_file); -static struct kobj_type bus_ktype = { +static void bus_release(struct kobject *kobj) +{ + struct subsys_private *priv = to_subsys_private(kobj); + + lockdep_unregister_key(&priv->lock_key); + kfree(priv); +} + +static const struct kobj_type bus_ktype = { .sysfs_ops = &bus_sysfs_ops, + .release = bus_release, }; -static int bus_uevent_filter(struct kset *kset, struct kobject *kobj) +static int bus_uevent_filter(const struct kobject *kobj) { - struct kobj_type *ktype = get_ktype(kobj); + const struct kobj_type *ktype = get_ktype(kobj); if (ktype == &bus_ktype) return 1; @@ -162,93 +231,94 @@ static const struct kset_uevent_ops bus_uevent_ops = { .filter = bus_uevent_filter, }; -static struct kset *bus_kset; - /* Manually detach a device from its associated driver. */ -static ssize_t driver_unbind(struct device_driver *drv, - const char *buf, size_t count) +static ssize_t unbind_store(struct device_driver *drv, const char *buf, + size_t count) { - struct bus_type *bus = bus_get(drv->bus); + const struct bus_type *bus = bus_get(drv->bus); struct device *dev; int err = -ENODEV; dev = bus_find_device_by_name(bus, NULL, buf); if (dev && dev->driver == drv) { - if (dev->parent) /* Needed for USB */ - device_lock(dev->parent); - device_release_driver(dev); - if (dev->parent) - device_unlock(dev->parent); + device_driver_detach(dev); err = count; } put_device(dev); bus_put(bus); return err; } -static DRIVER_ATTR(unbind, S_IWUSR, NULL, driver_unbind); +static DRIVER_ATTR_IGNORE_LOCKDEP(unbind, 0200, NULL, unbind_store); /* * Manually attach a device to a driver. * Note: the driver must want to bind to the device, * it is not possible to override the driver's id table. */ -static ssize_t driver_bind(struct device_driver *drv, - const char *buf, size_t count) +static ssize_t bind_store(struct device_driver *drv, const char *buf, + size_t count) { - struct bus_type *bus = bus_get(drv->bus); + const struct bus_type *bus = bus_get(drv->bus); struct device *dev; int err = -ENODEV; dev = bus_find_device_by_name(bus, NULL, buf); - if (dev && dev->driver == NULL && driver_match_device(drv, dev)) { - if (dev->parent) /* Needed for USB */ - device_lock(dev->parent); - device_lock(dev); - err = driver_probe_device(drv, dev); - device_unlock(dev); - if (dev->parent) - device_unlock(dev->parent); - - if (err > 0) { + if (dev && driver_match_device(drv, dev)) { + err = device_driver_attach(drv, dev); + if (!err) { /* success */ err = count; - } else if (err == 0) { - /* driver didn't accept device */ - err = -ENODEV; } } put_device(dev); bus_put(bus); return err; } -static DRIVER_ATTR(bind, S_IWUSR, NULL, driver_bind); +static DRIVER_ATTR_IGNORE_LOCKDEP(bind, 0200, NULL, bind_store); -static ssize_t show_drivers_autoprobe(struct bus_type *bus, char *buf) +static ssize_t drivers_autoprobe_show(const struct bus_type *bus, char *buf) { - return sprintf(buf, "%d\n", bus->p->drivers_autoprobe); + struct subsys_private *sp = bus_to_subsys(bus); + int ret; + + if (!sp) + return -EINVAL; + + ret = sysfs_emit(buf, "%d\n", sp->drivers_autoprobe); + subsys_put(sp); + return ret; } -static ssize_t store_drivers_autoprobe(struct bus_type *bus, +static ssize_t drivers_autoprobe_store(const struct bus_type *bus, const char *buf, size_t count) { + struct subsys_private *sp = bus_to_subsys(bus); + + if (!sp) + return -EINVAL; + if (buf[0] == '0') - bus->p->drivers_autoprobe = 0; + sp->drivers_autoprobe = 0; else - bus->p->drivers_autoprobe = 1; + sp->drivers_autoprobe = 1; + + subsys_put(sp); return count; } -static ssize_t store_drivers_probe(struct bus_type *bus, +static ssize_t drivers_probe_store(const struct bus_type *bus, const char *buf, size_t count) { struct device *dev; + int err = -EINVAL; dev = bus_find_device_by_name(bus, NULL, buf); if (!dev) return -ENODEV; - if (bus_rescan_devices_helper(dev, NULL) != 0) - return -EINVAL; - return count; + if (bus_rescan_devices_helper(dev, NULL) == 0) + err = count; + put_device(dev); + return err; } static struct device *next_device(struct klist_iter *i) @@ -264,6 +334,19 @@ static struct device *next_device(struct klist_iter *i) return dev; } +static struct device *prev_device(struct klist_iter *i) +{ + struct klist_node *n = klist_prev(i); + struct device *dev = NULL; + struct device_private *dev_prv; + + if (n) { + dev_prv = to_device_private_bus(n); + dev = dev_prv->device; + } + return dev; +} + /** * bus_for_each_dev - device iterator. * @bus: bus type. @@ -283,21 +366,23 @@ static struct device *next_device(struct klist_iter *i) * to retain this data, it should do so, and increment the reference * count in the supplied callback. */ -int bus_for_each_dev(struct bus_type *bus, struct device *start, - void *data, int (*fn)(struct device *, void *)) +int bus_for_each_dev(const struct bus_type *bus, struct device *start, + void *data, device_iter_t fn) { + struct subsys_private *sp = bus_to_subsys(bus); struct klist_iter i; struct device *dev; int error = 0; - if (!bus || !bus->p) + if (!sp) return -EINVAL; - klist_iter_init_node(&bus->p->klist_devices, &i, + klist_iter_init_node(&sp->klist_devices, &i, (start ? &start->p->knode_bus : NULL)); - while ((dev = next_device(&i)) && !error) + while (!error && (dev = next_device(&i))) error = fn(dev, data); klist_iter_exit(&i); + subsys_put(sp); return error; } EXPORT_SYMBOL_GPL(bus_for_each_dev); @@ -317,90 +402,55 @@ EXPORT_SYMBOL_GPL(bus_for_each_dev); * if it does. If the callback returns non-zero, this function will * return to the caller and not iterate over any more devices. */ -struct device *bus_find_device(struct bus_type *bus, - struct device *start, void *data, - int (*match)(struct device *dev, void *data)) +struct device *bus_find_device(const struct bus_type *bus, + struct device *start, const void *data, + device_match_t match) { + struct subsys_private *sp = bus_to_subsys(bus); struct klist_iter i; struct device *dev; - if (!bus || !bus->p) + if (!sp) return NULL; - klist_iter_init_node(&bus->p->klist_devices, &i, + klist_iter_init_node(&sp->klist_devices, &i, (start ? &start->p->knode_bus : NULL)); - while ((dev = next_device(&i))) - if (match(dev, data) && get_device(dev)) + while ((dev = next_device(&i))) { + if (match(dev, data)) { + get_device(dev); break; + } + } klist_iter_exit(&i); + subsys_put(sp); return dev; } EXPORT_SYMBOL_GPL(bus_find_device); -static int match_name(struct device *dev, void *data) -{ - const char *name = data; - - return sysfs_streq(name, dev_name(dev)); -} - -/** - * bus_find_device_by_name - device iterator for locating a particular device of a specific name - * @bus: bus type - * @start: Device to begin with - * @name: name of the device to match - * - * This is similar to the bus_find_device() function above, but it handles - * searching by a name automatically, no need to write another strcmp matching - * function. - */ -struct device *bus_find_device_by_name(struct bus_type *bus, - struct device *start, const char *name) -{ - return bus_find_device(bus, start, (void *)name, match_name); -} -EXPORT_SYMBOL_GPL(bus_find_device_by_name); - -/** - * subsys_find_device_by_id - find a device with a specific enumeration number - * @subsys: subsystem - * @id: index 'id' in struct device - * @hint: device to check first - * - * Check the hint's next object and if it is a match return it directly, - * otherwise, fall back to a full list search. Either way a reference for - * the returned object is taken. - */ -struct device *subsys_find_device_by_id(struct bus_type *subsys, unsigned int id, - struct device *hint) +struct device *bus_find_device_reverse(const struct bus_type *bus, + struct device *start, const void *data, + device_match_t match) { + struct subsys_private *sp = bus_to_subsys(bus); struct klist_iter i; struct device *dev; - if (!subsys) + if (!sp) return NULL; - if (hint) { - klist_iter_init_node(&subsys->p->klist_devices, &i, &hint->p->knode_bus); - dev = next_device(&i); - if (dev && dev->id == id && get_device(dev)) { - klist_iter_exit(&i); - return dev; - } - klist_iter_exit(&i); - } - - klist_iter_init_node(&subsys->p->klist_devices, &i, NULL); - while ((dev = next_device(&i))) { - if (dev->id == id && get_device(dev)) { - klist_iter_exit(&i); - return dev; + klist_iter_init_node(&sp->klist_devices, &i, + (start ? &start->p->knode_bus : NULL)); + while ((dev = prev_device(&i))) { + if (match(dev, data)) { + get_device(dev); + break; } } klist_iter_exit(&i); - return NULL; + subsys_put(sp); + return dev; } -EXPORT_SYMBOL_GPL(subsys_find_device_by_id); +EXPORT_SYMBOL_GPL(bus_find_device_reverse); static struct device_driver *next_driver(struct klist_iter *i) { @@ -433,54 +483,27 @@ static struct device_driver *next_driver(struct klist_iter *i) * in the callback. It must also be sure to increment the refcount * so it doesn't disappear before returning to the caller. */ -int bus_for_each_drv(struct bus_type *bus, struct device_driver *start, +int bus_for_each_drv(const struct bus_type *bus, struct device_driver *start, void *data, int (*fn)(struct device_driver *, void *)) { + struct subsys_private *sp = bus_to_subsys(bus); struct klist_iter i; struct device_driver *drv; int error = 0; - if (!bus) + if (!sp) return -EINVAL; - klist_iter_init_node(&bus->p->klist_drivers, &i, + klist_iter_init_node(&sp->klist_drivers, &i, start ? &start->p->knode_bus : NULL); while ((drv = next_driver(&i)) && !error) error = fn(drv, data); klist_iter_exit(&i); + subsys_put(sp); return error; } EXPORT_SYMBOL_GPL(bus_for_each_drv); -static int device_add_attrs(struct bus_type *bus, struct device *dev) -{ - int error = 0; - int i; - - if (!bus->dev_attrs) - return 0; - - for (i = 0; attr_name(bus->dev_attrs[i]); i++) { - error = device_create_file(dev, &bus->dev_attrs[i]); - if (error) { - while (--i >= 0) - device_remove_file(dev, &bus->dev_attrs[i]); - break; - } - } - return error; -} - -static void device_remove_attrs(struct bus_type *bus, struct device *dev) -{ - int i; - - if (bus->dev_attrs) { - for (i = 0; attr_name(bus->dev_attrs[i]); i++) - device_remove_file(dev, &bus->dev_attrs[i]); - } -} - /** * bus_add_device - add device to bus * @dev: device being added @@ -491,32 +514,46 @@ static void device_remove_attrs(struct bus_type *bus, struct device *dev) */ int bus_add_device(struct device *dev) { - struct bus_type *bus = bus_get(dev->bus); - int error = 0; + struct subsys_private *sp = bus_to_subsys(dev->bus); + int error; - if (bus) { - pr_debug("bus: '%s': add device %s\n", bus->name, dev_name(dev)); - error = device_add_attrs(bus, dev); - if (error) - goto out_put; - error = sysfs_create_link(&bus->p->devices_kset->kobj, - &dev->kobj, dev_name(dev)); - if (error) - goto out_id; - error = sysfs_create_link(&dev->kobj, - &dev->bus->p->subsys.kobj, "subsystem"); - if (error) - goto out_subsys; - klist_add_tail(&dev->p->knode_bus, &bus->p->klist_devices); + if (!sp) { + /* + * This is a normal operation for many devices that do not + * have a bus assigned to them, just say that all went + * well. + */ + return 0; } + + /* + * Reference in sp is now incremented and will be dropped when + * the device is removed from the bus + */ + + pr_debug("bus: '%s': add device %s\n", sp->bus->name, dev_name(dev)); + + error = device_add_groups(dev, sp->bus->dev_groups); + if (error) + goto out_put; + + error = sysfs_create_link(&sp->devices_kset->kobj, &dev->kobj, dev_name(dev)); + if (error) + goto out_groups; + + error = sysfs_create_link(&dev->kobj, &sp->subsys.kobj, "subsystem"); + if (error) + goto out_subsys; + + klist_add_tail(&dev->p->knode_bus, &sp->klist_devices); return 0; out_subsys: - sysfs_remove_link(&bus->p->devices_kset->kobj, dev_name(dev)); -out_id: - device_remove_attrs(bus, dev); + sysfs_remove_link(&sp->devices_kset->kobj, dev_name(dev)); +out_groups: + device_remove_groups(dev, sp->bus->dev_groups); out_put: - bus_put(dev->bus); + subsys_put(sp); return error; } @@ -528,23 +565,20 @@ out_put: */ void bus_probe_device(struct device *dev) { - struct bus_type *bus = dev->bus; + struct subsys_private *sp = bus_to_subsys(dev->bus); struct subsys_interface *sif; - int ret; - if (!bus) + if (!sp) return; - if (bus->p->drivers_autoprobe) { - ret = device_attach(dev); - WARN_ON(ret < 0); - } + device_initial_probe(dev); - mutex_lock(&bus->p->mutex); - list_for_each_entry(sif, &bus->p->interfaces, node) + mutex_lock(&sp->mutex); + list_for_each_entry(sif, &sp->interfaces, node) if (sif->add_dev) sif->add_dev(dev, sif); - mutex_unlock(&bus->p->mutex); + mutex_unlock(&sp->mutex); + subsys_put(sp); } /** @@ -559,60 +593,35 @@ void bus_probe_device(struct device *dev) */ void bus_remove_device(struct device *dev) { - struct bus_type *bus = dev->bus; + struct subsys_private *sp = bus_to_subsys(dev->bus); struct subsys_interface *sif; - if (!bus) + if (!sp) return; - mutex_lock(&bus->p->mutex); - list_for_each_entry(sif, &bus->p->interfaces, node) + mutex_lock(&sp->mutex); + list_for_each_entry(sif, &sp->interfaces, node) if (sif->remove_dev) sif->remove_dev(dev, sif); - mutex_unlock(&bus->p->mutex); + mutex_unlock(&sp->mutex); sysfs_remove_link(&dev->kobj, "subsystem"); - sysfs_remove_link(&dev->bus->p->devices_kset->kobj, - dev_name(dev)); - device_remove_attrs(dev->bus, dev); + sysfs_remove_link(&sp->devices_kset->kobj, dev_name(dev)); + device_remove_groups(dev, dev->bus->dev_groups); if (klist_node_attached(&dev->p->knode_bus)) klist_del(&dev->p->knode_bus); pr_debug("bus: '%s': remove device %s\n", dev->bus->name, dev_name(dev)); device_release_driver(dev); - bus_put(dev->bus); -} - -static int driver_add_attrs(struct bus_type *bus, struct device_driver *drv) -{ - int error = 0; - int i; - - if (bus->drv_attrs) { - for (i = 0; attr_name(bus->drv_attrs[i]); i++) { - error = driver_create_file(drv, &bus->drv_attrs[i]); - if (error) - goto err; - } - } -done: - return error; -err: - while (--i >= 0) - driver_remove_file(drv, &bus->drv_attrs[i]); - goto done; -} - -static void driver_remove_attrs(struct bus_type *bus, - struct device_driver *drv) -{ - int i; - if (bus->drv_attrs) { - for (i = 0; attr_name(bus->drv_attrs[i]); i++) - driver_remove_file(drv, &bus->drv_attrs[i]); - } + /* + * Decrement the reference count twice, once for the bus_to_subsys() + * call in the start of this function, and the second one from the + * reference increment in bus_add_device() + */ + subsys_put(sp); + subsys_put(sp); } static int __must_check add_bind_files(struct device_driver *drv) @@ -634,11 +643,10 @@ static void remove_bind_files(struct device_driver *drv) driver_remove_file(drv, &driver_attr_unbind); } -static BUS_ATTR(drivers_probe, S_IWUSR, NULL, store_drivers_probe); -static BUS_ATTR(drivers_autoprobe, S_IWUSR | S_IRUGO, - show_drivers_autoprobe, store_drivers_autoprobe); +static BUS_ATTR_WO(drivers_probe); +static BUS_ATTR_RW(drivers_autoprobe); -static int add_probe_files(struct bus_type *bus) +static int add_probe_files(const struct bus_type *bus) { int retval; @@ -653,22 +661,21 @@ out: return retval; } -static void remove_probe_files(struct bus_type *bus) +static void remove_probe_files(const struct bus_type *bus) { bus_remove_file(bus, &bus_attr_drivers_autoprobe); bus_remove_file(bus, &bus_attr_drivers_probe); } -static ssize_t driver_uevent_store(struct device_driver *drv, - const char *buf, size_t count) +static ssize_t uevent_store(struct device_driver *drv, const char *buf, + size_t count) { - enum kobject_action action; + int rc; - if (kobject_action_type(buf, count, &action) == 0) - kobject_uevent(&drv->p->kobj, action); - return count; + rc = kobject_synth_uevent(&drv->p->kobj, buf, count); + return rc ? rc : count; } -static DRIVER_ATTR(uevent, S_IWUSR, NULL, driver_uevent_store); +static DRIVER_ATTR_WO(uevent); /** * bus_add_driver - Add a driver to the bus. @@ -676,15 +683,18 @@ static DRIVER_ATTR(uevent, S_IWUSR, NULL, driver_uevent_store); */ int bus_add_driver(struct device_driver *drv) { - struct bus_type *bus; + struct subsys_private *sp = bus_to_subsys(drv->bus); struct driver_private *priv; int error = 0; - bus = bus_get(drv->bus); - if (!bus) + if (!sp) return -EINVAL; - pr_debug("bus: '%s': add driver %s\n", bus->name, drv->name); + /* + * Reference in sp is now incremented and will be dropped when + * the driver is removed from the bus + */ + pr_debug("bus: '%s': add driver %s\n", sp->bus->name, drv->name); priv = kzalloc(sizeof(*priv), GFP_KERNEL); if (!priv) { @@ -694,29 +704,34 @@ int bus_add_driver(struct device_driver *drv) klist_init(&priv->klist_devices, NULL, NULL); priv->driver = drv; drv->p = priv; - priv->kobj.kset = bus->p->drivers_kset; + priv->kobj.kset = sp->drivers_kset; error = kobject_init_and_add(&priv->kobj, &driver_ktype, NULL, "%s", drv->name); if (error) goto out_unregister; - klist_add_tail(&priv->knode_bus, &bus->p->klist_drivers); - if (drv->bus->p->drivers_autoprobe) { + klist_add_tail(&priv->knode_bus, &sp->klist_drivers); + if (sp->drivers_autoprobe) { error = driver_attach(drv); if (error) - goto out_unregister; + goto out_del_list; + } + error = module_add_driver(drv->owner, drv); + if (error) { + printk(KERN_ERR "%s: failed to create module links for %s\n", + __func__, drv->name); + goto out_detach; } - module_add_driver(drv->owner, drv); error = driver_create_file(drv, &driver_attr_uevent); if (error) { printk(KERN_ERR "%s: uevent attr (%s) failed\n", __func__, drv->name); } - error = driver_add_attrs(bus, drv); + error = driver_add_groups(drv, sp->bus->drv_groups); if (error) { /* How the hell do we get out of this pickle? Give up */ - printk(KERN_ERR "%s: driver_add_attrs(%s) failed\n", + printk(KERN_ERR "%s: driver_add_groups(%s) failed\n", __func__, drv->name); } @@ -731,12 +746,16 @@ int bus_add_driver(struct device_driver *drv) return 0; +out_detach: + driver_detach(drv); +out_del_list: + klist_del(&priv->knode_bus); out_unregister: kobject_put(&priv->kobj); - kfree(drv->p); + /* drv->p is freed in driver_release() */ drv->p = NULL; out_put_bus: - bus_put(bus); + subsys_put(sp); return error; } @@ -750,19 +769,29 @@ out_put_bus: */ void bus_remove_driver(struct device_driver *drv) { - if (!drv->bus) + struct subsys_private *sp = bus_to_subsys(drv->bus); + + if (!sp) return; + pr_debug("bus: '%s': remove driver %s\n", sp->bus->name, drv->name); + if (!drv->suppress_bind_attrs) remove_bind_files(drv); - driver_remove_attrs(drv->bus, drv); + driver_remove_groups(drv, sp->bus->drv_groups); driver_remove_file(drv, &driver_attr_uevent); klist_remove(&drv->p->knode_bus); - pr_debug("bus: '%s': remove driver %s\n", drv->bus->name, drv->name); driver_detach(drv); module_remove_driver(drv); kobject_put(&drv->p->kobj); - bus_put(drv->bus); + + /* + * Decrement the reference count twice, once for the bus_to_subsys() + * call in the start of this function, and the second one from the + * reference increment in bus_add_driver() + */ + subsys_put(sp); + subsys_put(sp); } /* Helper for bus_rescan_devices's iter */ @@ -772,10 +801,10 @@ static int __must_check bus_rescan_devices_helper(struct device *dev, int ret = 0; if (!dev->driver) { - if (dev->parent) /* Needed for USB */ + if (dev->parent && dev->bus->need_parent_lock) device_lock(dev->parent); ret = device_attach(dev); - if (dev->parent) + if (dev->parent && dev->bus->need_parent_lock) device_unlock(dev->parent); } return ret < 0 ? ret : 0; @@ -789,7 +818,7 @@ static int __must_check bus_rescan_devices_helper(struct device *dev, * attached and rescan it against existing drivers to see if it matches * any by calling device_attach() for the unbound devices. */ -int bus_rescan_devices(struct bus_type *bus) +int bus_rescan_devices(const struct bus_type *bus) { return bus_for_each_dev(bus, NULL, NULL, bus_rescan_devices_helper); } @@ -806,70 +835,12 @@ EXPORT_SYMBOL_GPL(bus_rescan_devices); */ int device_reprobe(struct device *dev) { - if (dev->driver) { - if (dev->parent) /* Needed for USB */ - device_lock(dev->parent); - device_release_driver(dev); - if (dev->parent) - device_unlock(dev->parent); - } + if (dev->driver) + device_driver_detach(dev); return bus_rescan_devices_helper(dev, NULL); } EXPORT_SYMBOL_GPL(device_reprobe); -/** - * find_bus - locate bus by name. - * @name: name of bus. - * - * Call kset_find_obj() to iterate over list of buses to - * find a bus by name. Return bus if found. - * - * Note that kset_find_obj increments bus' reference count. - */ -#if 0 -struct bus_type *find_bus(char *name) -{ - struct kobject *k = kset_find_obj(bus_kset, name); - return k ? to_bus(k) : NULL; -} -#endif /* 0 */ - - -/** - * bus_add_attrs - Add default attributes for this bus. - * @bus: Bus that has just been registered. - */ - -static int bus_add_attrs(struct bus_type *bus) -{ - int error = 0; - int i; - - if (bus->bus_attrs) { - for (i = 0; attr_name(bus->bus_attrs[i]); i++) { - error = bus_create_file(bus, &bus->bus_attrs[i]); - if (error) - goto err; - } - } -done: - return error; -err: - while (--i >= 0) - bus_remove_file(bus, &bus->bus_attrs[i]); - goto done; -} - -static void bus_remove_attrs(struct bus_type *bus) -{ - int i; - - if (bus->bus_attrs) { - for (i = 0; attr_name(bus->bus_attrs[i]); i++) - bus_remove_file(bus, &bus->bus_attrs[i]); - } -} - static void klist_devices_get(struct klist_node *n) { struct device_private *dev_prv = to_device_private_bus(n); @@ -886,16 +857,30 @@ static void klist_devices_put(struct klist_node *n) put_device(dev); } -static ssize_t bus_uevent_store(struct bus_type *bus, +static ssize_t bus_uevent_store(const struct bus_type *bus, const char *buf, size_t count) { - enum kobject_action action; + struct subsys_private *sp = bus_to_subsys(bus); + int ret; + + if (!sp) + return -EINVAL; + + ret = kobject_synth_uevent(&sp->subsys.kobj, buf, count); + subsys_put(sp); - if (kobject_action_type(buf, count, &action) == 0) - kobject_uevent(&bus->p->subsys.kobj, action); + if (ret) + return ret; return count; } -static BUS_ATTR(uevent, S_IWUSR, NULL, bus_uevent_store); +/* + * "open code" the old BUS_ATTR() macro here. We want to use BUS_ATTR_WO() + * here, but can not use it as earlier in the file we have + * DEVICE_ATTR_WO(uevent), which would cause a clash with the with the store + * function name. + */ +static struct bus_attribute bus_attr_uevent = __ATTR(uevent, 0200, NULL, + bus_uevent_store); /** * bus_register - register a driver-core subsystem @@ -905,27 +890,28 @@ static BUS_ATTR(uevent, S_IWUSR, NULL, bus_uevent_store); * infrastructure, then register the children subsystems it has: * the devices and drivers that belong to the subsystem. */ -int bus_register(struct bus_type *bus) +int bus_register(const struct bus_type *bus) { int retval; struct subsys_private *priv; - struct lock_class_key *key = &bus->lock_key; + struct kobject *bus_kobj; + struct lock_class_key *key; priv = kzalloc(sizeof(struct subsys_private), GFP_KERNEL); if (!priv) return -ENOMEM; priv->bus = bus; - bus->p = priv; BLOCKING_INIT_NOTIFIER_HEAD(&priv->bus_notifier); - retval = kobject_set_name(&priv->subsys.kobj, "%s", bus->name); + bus_kobj = &priv->subsys.kobj; + retval = kobject_set_name(bus_kobj, "%s", bus->name); if (retval) goto out; - priv->subsys.kobj.kset = bus_kset; - priv->subsys.kobj.ktype = &bus_ktype; + bus_kobj->kset = bus_kset; + bus_kobj->ktype = &bus_ktype; priv->drivers_autoprobe = 1; retval = kset_register(&priv->subsys); @@ -936,21 +922,21 @@ int bus_register(struct bus_type *bus) if (retval) goto bus_uevent_fail; - priv->devices_kset = kset_create_and_add("devices", NULL, - &priv->subsys.kobj); + priv->devices_kset = kset_create_and_add("devices", NULL, bus_kobj); if (!priv->devices_kset) { retval = -ENOMEM; goto bus_devices_fail; } - priv->drivers_kset = kset_create_and_add("drivers", NULL, - &priv->subsys.kobj); + priv->drivers_kset = kset_create_and_add("drivers", NULL, bus_kobj); if (!priv->drivers_kset) { retval = -ENOMEM; goto bus_drivers_fail; } INIT_LIST_HEAD(&priv->interfaces); + key = &priv->lock_key; + lockdep_register_key(key); __mutex_init(&priv->mutex, "subsys mutex", key); klist_init(&priv->klist_devices, klist_devices_get, klist_devices_put); klist_init(&priv->klist_drivers, NULL, NULL); @@ -959,26 +945,27 @@ int bus_register(struct bus_type *bus) if (retval) goto bus_probe_files_fail; - retval = bus_add_attrs(bus); + retval = sysfs_create_groups(bus_kobj, bus->bus_groups); if (retval) - goto bus_attrs_fail; + goto bus_groups_fail; pr_debug("bus: '%s': registered\n", bus->name); return 0; -bus_attrs_fail: +bus_groups_fail: remove_probe_files(bus); bus_probe_files_fail: - kset_unregister(bus->p->drivers_kset); + kset_unregister(priv->drivers_kset); bus_drivers_fail: - kset_unregister(bus->p->devices_kset); + kset_unregister(priv->devices_kset); bus_devices_fail: bus_remove_file(bus, &bus_attr_uevent); bus_uevent_fail: - kset_unregister(&bus->p->subsys); + kset_unregister(&priv->subsys); + /* Above kset_unregister() will kfree @priv */ + priv = NULL; out: - kfree(bus->p); - bus->p = NULL; + kfree(priv); return retval; } EXPORT_SYMBOL_GPL(bus_register); @@ -990,45 +977,82 @@ EXPORT_SYMBOL_GPL(bus_register); * Unregister the child subsystems and the bus itself. * Finally, we call bus_put() to release the refcount */ -void bus_unregister(struct bus_type *bus) +void bus_unregister(const struct bus_type *bus) { + struct subsys_private *sp = bus_to_subsys(bus); + struct kobject *bus_kobj; + + if (!sp) + return; + pr_debug("bus: '%s': unregistering\n", bus->name); - if (bus->dev_root) - device_unregister(bus->dev_root); - bus_remove_attrs(bus); + if (sp->dev_root) + device_unregister(sp->dev_root); + + bus_kobj = &sp->subsys.kobj; + sysfs_remove_groups(bus_kobj, bus->bus_groups); remove_probe_files(bus); - kset_unregister(bus->p->drivers_kset); - kset_unregister(bus->p->devices_kset); bus_remove_file(bus, &bus_attr_uevent); - kset_unregister(&bus->p->subsys); - kfree(bus->p); - bus->p = NULL; + + kset_unregister(sp->drivers_kset); + kset_unregister(sp->devices_kset); + kset_unregister(&sp->subsys); + subsys_put(sp); } EXPORT_SYMBOL_GPL(bus_unregister); -int bus_register_notifier(struct bus_type *bus, struct notifier_block *nb) +int bus_register_notifier(const struct bus_type *bus, struct notifier_block *nb) { - return blocking_notifier_chain_register(&bus->p->bus_notifier, nb); + struct subsys_private *sp = bus_to_subsys(bus); + int retval; + + if (!sp) + return -EINVAL; + + retval = blocking_notifier_chain_register(&sp->bus_notifier, nb); + subsys_put(sp); + return retval; } EXPORT_SYMBOL_GPL(bus_register_notifier); -int bus_unregister_notifier(struct bus_type *bus, struct notifier_block *nb) +int bus_unregister_notifier(const struct bus_type *bus, struct notifier_block *nb) { - return blocking_notifier_chain_unregister(&bus->p->bus_notifier, nb); + struct subsys_private *sp = bus_to_subsys(bus); + int retval; + + if (!sp) + return -EINVAL; + retval = blocking_notifier_chain_unregister(&sp->bus_notifier, nb); + subsys_put(sp); + return retval; } EXPORT_SYMBOL_GPL(bus_unregister_notifier); -struct kset *bus_get_kset(struct bus_type *bus) +void bus_notify(struct device *dev, enum bus_notifier_event value) { - return &bus->p->subsys; + struct subsys_private *sp = bus_to_subsys(dev->bus); + + if (!sp) + return; + + blocking_notifier_call_chain(&sp->bus_notifier, value, dev); + subsys_put(sp); } -EXPORT_SYMBOL_GPL(bus_get_kset); -struct klist *bus_get_device_klist(struct bus_type *bus) +struct kset *bus_get_kset(const struct bus_type *bus) { - return &bus->p->klist_devices; + struct subsys_private *sp = bus_to_subsys(bus); + struct kset *kset; + + if (!sp) + return NULL; + + kset = &sp->subsys; + subsys_put(sp); + + return kset; } -EXPORT_SYMBOL_GPL(bus_get_device_klist); +EXPORT_SYMBOL_GPL(bus_get_kset); /* * Yes, this forcibly breaks the klist abstraction temporarily. It @@ -1041,13 +1065,11 @@ static void device_insertion_sort_klist(struct device *a, struct list_head *list int (*compare)(const struct device *a, const struct device *b)) { - struct list_head *pos; struct klist_node *n; struct device_private *dev_prv; struct device *b; - list_for_each(pos, list) { - n = container_of(pos, struct klist_node, n_node); + list_for_each_entry(n, list, n_node) { dev_prv = to_device_private_bus(n); b = dev_prv->device; if (compare(a, b) <= 0) { @@ -1059,35 +1081,42 @@ static void device_insertion_sort_klist(struct device *a, struct list_head *list list_move_tail(&a->p->knode_bus.n_node, list); } -void bus_sort_breadthfirst(struct bus_type *bus, +void bus_sort_breadthfirst(const struct bus_type *bus, int (*compare)(const struct device *a, const struct device *b)) { + struct subsys_private *sp = bus_to_subsys(bus); LIST_HEAD(sorted_devices); - struct list_head *pos, *tmp; - struct klist_node *n; + struct klist_node *n, *tmp; struct device_private *dev_prv; struct device *dev; struct klist *device_klist; - device_klist = bus_get_device_klist(bus); + if (!sp) + return; + device_klist = &sp->klist_devices; spin_lock(&device_klist->k_lock); - list_for_each_safe(pos, tmp, &device_klist->k_list) { - n = container_of(pos, struct klist_node, n_node); + list_for_each_entry_safe(n, tmp, &device_klist->k_list, n_node) { dev_prv = to_device_private_bus(n); dev = dev_prv->device; device_insertion_sort_klist(dev, &sorted_devices, compare); } list_splice(&sorted_devices, &device_klist->k_list); spin_unlock(&device_klist->k_lock); + subsys_put(sp); } EXPORT_SYMBOL_GPL(bus_sort_breadthfirst); +struct subsys_dev_iter { + struct klist_iter ki; + const struct device_type *type; +}; + /** * subsys_dev_iter_init - initialize subsys device iterator * @iter: subsys iterator to initialize - * @subsys: the subsys we wanna iterate over + * @sp: the subsys private (i.e. bus) we wanna iterate over * @start: the device to start iterating from, if any * @type: device_type of the devices to iterate over, NULL for all * @@ -1096,17 +1125,16 @@ EXPORT_SYMBOL_GPL(bus_sort_breadthfirst); * otherwise if it is NULL, the iteration starts at the beginning of * the list. */ -void subsys_dev_iter_init(struct subsys_dev_iter *iter, struct bus_type *subsys, - struct device *start, const struct device_type *type) +static void subsys_dev_iter_init(struct subsys_dev_iter *iter, struct subsys_private *sp, + struct device *start, const struct device_type *type) { struct klist_node *start_knode = NULL; if (start) start_knode = &start->p->knode_bus; - klist_iter_init_node(&subsys->p->klist_devices, &iter->ki, start_knode); + klist_iter_init_node(&sp->klist_devices, &iter->ki, start_knode); iter->type = type; } -EXPORT_SYMBOL_GPL(subsys_dev_iter_init); /** * subsys_dev_iter_next - iterate to the next device @@ -1120,7 +1148,7 @@ EXPORT_SYMBOL_GPL(subsys_dev_iter_init); * free to do whatever it wants to do with the device including * calling back into subsys code. */ -struct device *subsys_dev_iter_next(struct subsys_dev_iter *iter) +static struct device *subsys_dev_iter_next(struct subsys_dev_iter *iter) { struct klist_node *knode; struct device *dev; @@ -1129,12 +1157,11 @@ struct device *subsys_dev_iter_next(struct subsys_dev_iter *iter) knode = klist_next(&iter->ki); if (!knode) return NULL; - dev = container_of(knode, struct device_private, knode_bus)->device; + dev = to_device_private_bus(knode)->device; if (!iter->type || iter->type == dev->type) return dev; } } -EXPORT_SYMBOL_GPL(subsys_dev_iter_next); /** * subsys_dev_iter_exit - finish iteration @@ -1143,34 +1170,38 @@ EXPORT_SYMBOL_GPL(subsys_dev_iter_next); * Finish an iteration. Always call this function after iteration is * complete whether the iteration ran till the end or not. */ -void subsys_dev_iter_exit(struct subsys_dev_iter *iter) +static void subsys_dev_iter_exit(struct subsys_dev_iter *iter) { klist_iter_exit(&iter->ki); } -EXPORT_SYMBOL_GPL(subsys_dev_iter_exit); int subsys_interface_register(struct subsys_interface *sif) { - struct bus_type *subsys; + struct subsys_private *sp; struct subsys_dev_iter iter; struct device *dev; if (!sif || !sif->subsys) return -ENODEV; - subsys = bus_get(sif->subsys); - if (!subsys) + sp = bus_to_subsys(sif->subsys); + if (!sp) return -EINVAL; - mutex_lock(&subsys->p->mutex); - list_add_tail(&sif->node, &subsys->p->interfaces); + /* + * Reference in sp is now incremented and will be dropped when + * the interface is removed from the bus + */ + + mutex_lock(&sp->mutex); + list_add_tail(&sif->node, &sp->interfaces); if (sif->add_dev) { - subsys_dev_iter_init(&iter, subsys, NULL, NULL); + subsys_dev_iter_init(&iter, sp, NULL, NULL); while ((dev = subsys_dev_iter_next(&iter))) sif->add_dev(dev, sif); subsys_dev_iter_exit(&iter); } - mutex_unlock(&subsys->p->mutex); + mutex_unlock(&sp->mutex); return 0; } @@ -1178,26 +1209,34 @@ EXPORT_SYMBOL_GPL(subsys_interface_register); void subsys_interface_unregister(struct subsys_interface *sif) { - struct bus_type *subsys; + struct subsys_private *sp; struct subsys_dev_iter iter; struct device *dev; if (!sif || !sif->subsys) return; - subsys = sif->subsys; + sp = bus_to_subsys(sif->subsys); + if (!sp) + return; - mutex_lock(&subsys->p->mutex); + mutex_lock(&sp->mutex); list_del_init(&sif->node); if (sif->remove_dev) { - subsys_dev_iter_init(&iter, subsys, NULL, NULL); + subsys_dev_iter_init(&iter, sp, NULL, NULL); while ((dev = subsys_dev_iter_next(&iter))) sif->remove_dev(dev, sif); subsys_dev_iter_exit(&iter); } - mutex_unlock(&subsys->p->mutex); + mutex_unlock(&sp->mutex); - bus_put(subsys); + /* + * Decrement the reference count twice, once for the bus_to_subsys() + * call in the start of this function, and the second one from the + * reference increment in subsys_interface_register() + */ + subsys_put(sp); + subsys_put(sp); } EXPORT_SYMBOL_GPL(subsys_interface_unregister); @@ -1206,10 +1245,11 @@ static void system_root_device_release(struct device *dev) kfree(dev); } -static int subsys_register(struct bus_type *subsys, +static int subsys_register(const struct bus_type *subsys, const struct attribute_group **groups, struct kobject *parent_of_root) { + struct subsys_private *sp; struct device *dev; int err; @@ -1217,6 +1257,12 @@ static int subsys_register(struct bus_type *subsys, if (err < 0) return err; + sp = bus_to_subsys(subsys); + if (!sp) { + err = -EINVAL; + goto err_sp; + } + dev = kzalloc(sizeof(struct device), GFP_KERNEL); if (!dev) { err = -ENOMEM; @@ -1235,7 +1281,8 @@ static int subsys_register(struct bus_type *subsys, if (err < 0) goto err_dev_reg; - subsys->dev_root = dev; + sp->dev_root = dev; + subsys_put(sp); return 0; err_dev_reg: @@ -1244,6 +1291,8 @@ err_dev_reg: err_name: kfree(dev); err_dev: + subsys_put(sp); +err_sp: bus_unregister(subsys); return err; } @@ -1257,7 +1306,7 @@ err_dev: * with the name of the subsystem. The root device can carry subsystem- * wide attributes. All registered devices are below this single root * device and are named after the subsystem with a simple enumeration - * number appended. The registered devices are not explicitely named; + * number appended. The registered devices are not explicitly named; * only 'id' in the device needs to be set. * * Do not use this interface for anything new, it exists for compatibility @@ -1266,7 +1315,7 @@ err_dev: * directory itself and not some create fake root-device placed in * /sys/devices/system/<name>. */ -int subsys_system_register(struct bus_type *subsys, +int subsys_system_register(const struct bus_type *subsys, const struct attribute_group **groups) { return subsys_register(subsys, groups, &system_kset->kobj); @@ -1279,17 +1328,17 @@ EXPORT_SYMBOL_GPL(subsys_system_register); * @groups: default attributes for the root device * * All 'virtual' subsystems have a /sys/devices/system/<name> root device - * with the name of the subystem. The root device can carry subsystem-wide + * with the name of the subsystem. The root device can carry subsystem-wide * attributes. All registered devices are below this single root device. * There's no restriction on device naming. This is for kernel software * constructs which need sysfs interface. */ -int subsys_virtual_register(struct bus_type *subsys, +int subsys_virtual_register(const struct bus_type *subsys, const struct attribute_group **groups) { struct kobject *virtual_dir; - virtual_dir = virtual_device_parent(NULL); + virtual_dir = virtual_device_parent(); if (!virtual_dir) return -ENOMEM; @@ -1297,6 +1346,82 @@ int subsys_virtual_register(struct bus_type *subsys, } EXPORT_SYMBOL_GPL(subsys_virtual_register); +/** + * driver_find - locate driver on a bus by its name. + * @name: name of the driver. + * @bus: bus to scan for the driver. + * + * Call kset_find_obj() to iterate over list of drivers on + * a bus to find driver by name. Return driver if found. + * + * This routine provides no locking to prevent the driver it returns + * from being unregistered or unloaded while the caller is using it. + * The caller is responsible for preventing this. + */ +struct device_driver *driver_find(const char *name, const struct bus_type *bus) +{ + struct subsys_private *sp = bus_to_subsys(bus); + struct kobject *k; + struct driver_private *priv; + + if (!sp) + return NULL; + + k = kset_find_obj(sp->drivers_kset, name); + subsys_put(sp); + if (!k) + return NULL; + + priv = to_driver(k); + + /* Drop reference added by kset_find_obj() */ + kobject_put(k); + return priv->driver; +} +EXPORT_SYMBOL_GPL(driver_find); + +/* + * Warning, the value could go to "removed" instantly after calling this function, so be very + * careful when calling it... + */ +bool bus_is_registered(const struct bus_type *bus) +{ + struct subsys_private *sp = bus_to_subsys(bus); + bool is_initialized = false; + + if (sp) { + is_initialized = true; + subsys_put(sp); + } + return is_initialized; +} + +/** + * bus_get_dev_root - return a pointer to the "device root" of a bus + * @bus: bus to return the device root of. + * + * If a bus has a "device root" structure, return it, WITH THE REFERENCE + * COUNT INCREMENTED. + * + * Note, when finished with the device, a call to put_device() is required. + * + * If the device root is not present (or bus is not a valid pointer), NULL + * will be returned. + */ +struct device *bus_get_dev_root(const struct bus_type *bus) +{ + struct subsys_private *sp = bus_to_subsys(bus); + struct device *dev_root; + + if (!sp) + return NULL; + + dev_root = get_device(sp->dev_root); + subsys_put(sp); + return dev_root; +} +EXPORT_SYMBOL_GPL(bus_get_dev_root); + int __init buses_init(void) { bus_kset = kset_create_and_add("bus", &bus_uevent_ops, NULL); @@ -1304,8 +1429,13 @@ int __init buses_init(void) return -ENOMEM; system_kset = kset_create_and_add("system", NULL, &devices_kset->kobj); - if (!system_kset) + if (!system_kset) { + /* Do error handling here as devices_init() do */ + kset_unregister(bus_kset); + bus_kset = NULL; + pr_err("%s: failed to create and add kset 'bus'\n", __func__); return -ENOMEM; + } return 0; } diff --git a/drivers/base/cacheinfo.c b/drivers/base/cacheinfo.c new file mode 100644 index 000000000000..613410705a47 --- /dev/null +++ b/drivers/base/cacheinfo.c @@ -0,0 +1,1047 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * cacheinfo support - processor cache information via sysfs + * + * Based on arch/x86/kernel/cpu/intel_cacheinfo.c + * Author: Sudeep Holla <sudeep.holla@arm.com> + */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/acpi.h> +#include <linux/bitfield.h> +#include <linux/bitops.h> +#include <linux/cacheinfo.h> +#include <linux/compiler.h> +#include <linux/cpu.h> +#include <linux/device.h> +#include <linux/init.h> +#include <linux/of.h> +#include <linux/sched.h> +#include <linux/slab.h> +#include <linux/smp.h> +#include <linux/sysfs.h> + +/* pointer to per cpu cacheinfo */ +static DEFINE_PER_CPU(struct cpu_cacheinfo, ci_cpu_cacheinfo); +#define ci_cacheinfo(cpu) (&per_cpu(ci_cpu_cacheinfo, cpu)) +#define cache_leaves(cpu) (ci_cacheinfo(cpu)->num_leaves) +#define per_cpu_cacheinfo(cpu) (ci_cacheinfo(cpu)->info_list) +#define per_cpu_cacheinfo_idx(cpu, idx) \ + (per_cpu_cacheinfo(cpu) + (idx)) + +/* Set if no cache information is found in DT/ACPI. */ +static bool use_arch_info; + +struct cpu_cacheinfo *get_cpu_cacheinfo(unsigned int cpu) +{ + return ci_cacheinfo(cpu); +} + +static inline bool cache_leaves_are_shared(struct cacheinfo *this_leaf, + struct cacheinfo *sib_leaf) +{ + /* + * For non DT/ACPI systems, assume unique level 1 caches, + * system-wide shared caches for all other levels. + */ + if (!(IS_ENABLED(CONFIG_OF) || IS_ENABLED(CONFIG_ACPI)) || + use_arch_info) + return (this_leaf->level != 1) && (sib_leaf->level != 1); + + if ((sib_leaf->attributes & CACHE_ID) && + (this_leaf->attributes & CACHE_ID)) + return sib_leaf->id == this_leaf->id; + + return sib_leaf->fw_token == this_leaf->fw_token; +} + +bool last_level_cache_is_valid(unsigned int cpu) +{ + struct cacheinfo *llc; + + if (!cache_leaves(cpu) || !per_cpu_cacheinfo(cpu)) + return false; + + llc = per_cpu_cacheinfo_idx(cpu, cache_leaves(cpu) - 1); + + return (llc->attributes & CACHE_ID) || !!llc->fw_token; + +} + +bool last_level_cache_is_shared(unsigned int cpu_x, unsigned int cpu_y) +{ + struct cacheinfo *llc_x, *llc_y; + + if (!last_level_cache_is_valid(cpu_x) || + !last_level_cache_is_valid(cpu_y)) + return false; + + llc_x = per_cpu_cacheinfo_idx(cpu_x, cache_leaves(cpu_x) - 1); + llc_y = per_cpu_cacheinfo_idx(cpu_y, cache_leaves(cpu_y) - 1); + + return cache_leaves_are_shared(llc_x, llc_y); +} + +#ifdef CONFIG_OF + +static bool of_check_cache_nodes(struct device_node *np); + +/* OF properties to query for a given cache type */ +struct cache_type_info { + const char *size_prop; + const char *line_size_props[2]; + const char *nr_sets_prop; +}; + +static const struct cache_type_info cache_type_info[] = { + { + .size_prop = "cache-size", + .line_size_props = { "cache-line-size", + "cache-block-size", }, + .nr_sets_prop = "cache-sets", + }, { + .size_prop = "i-cache-size", + .line_size_props = { "i-cache-line-size", + "i-cache-block-size", }, + .nr_sets_prop = "i-cache-sets", + }, { + .size_prop = "d-cache-size", + .line_size_props = { "d-cache-line-size", + "d-cache-block-size", }, + .nr_sets_prop = "d-cache-sets", + }, +}; + +static inline int get_cacheinfo_idx(enum cache_type type) +{ + if (type == CACHE_TYPE_UNIFIED) + return 0; + return type; +} + +static void cache_size(struct cacheinfo *this_leaf, struct device_node *np) +{ + const char *propname; + int ct_idx; + + ct_idx = get_cacheinfo_idx(this_leaf->type); + propname = cache_type_info[ct_idx].size_prop; + + of_property_read_u32(np, propname, &this_leaf->size); +} + +/* not cache_line_size() because that's a macro in include/linux/cache.h */ +static void cache_get_line_size(struct cacheinfo *this_leaf, + struct device_node *np) +{ + int i, lim, ct_idx; + + ct_idx = get_cacheinfo_idx(this_leaf->type); + lim = ARRAY_SIZE(cache_type_info[ct_idx].line_size_props); + + for (i = 0; i < lim; i++) { + int ret; + u32 line_size; + const char *propname; + + propname = cache_type_info[ct_idx].line_size_props[i]; + ret = of_property_read_u32(np, propname, &line_size); + if (!ret) { + this_leaf->coherency_line_size = line_size; + break; + } + } +} + +static void cache_nr_sets(struct cacheinfo *this_leaf, struct device_node *np) +{ + const char *propname; + int ct_idx; + + ct_idx = get_cacheinfo_idx(this_leaf->type); + propname = cache_type_info[ct_idx].nr_sets_prop; + + of_property_read_u32(np, propname, &this_leaf->number_of_sets); +} + +static void cache_associativity(struct cacheinfo *this_leaf) +{ + unsigned int line_size = this_leaf->coherency_line_size; + unsigned int nr_sets = this_leaf->number_of_sets; + unsigned int size = this_leaf->size; + + /* + * If the cache is fully associative, there is no need to + * check the other properties. + */ + if (!(nr_sets == 1) && (nr_sets > 0 && size > 0 && line_size > 0)) + this_leaf->ways_of_associativity = (size / nr_sets) / line_size; +} + +static bool cache_node_is_unified(struct cacheinfo *this_leaf, + struct device_node *np) +{ + return of_property_read_bool(np, "cache-unified"); +} + +static bool match_cache_node(struct device_node *cpu, + const struct device_node *cache_node) +{ + struct device_node *prev, *cache = of_find_next_cache_node(cpu); + + while (cache) { + if (cache == cache_node) { + of_node_put(cache); + return true; + } + + prev = cache; + cache = of_find_next_cache_node(cache); + of_node_put(prev); + } + + return false; +} + +#ifndef arch_compact_of_hwid +#define arch_compact_of_hwid(_x) (_x) +#endif + +static void cache_of_set_id(struct cacheinfo *this_leaf, + struct device_node *cache_node) +{ + struct device_node *cpu; + u32 min_id = ~0; + + for_each_of_cpu_node(cpu) { + u64 id = of_get_cpu_hwid(cpu, 0); + + id = arch_compact_of_hwid(id); + if (FIELD_GET(GENMASK_ULL(63, 32), id)) { + of_node_put(cpu); + return; + } + + if (match_cache_node(cpu, cache_node)) + min_id = min(min_id, id); + } + + if (min_id != ~0) { + this_leaf->id = min_id; + this_leaf->attributes |= CACHE_ID; + } +} + +static void cache_of_set_props(struct cacheinfo *this_leaf, + struct device_node *np) +{ + /* + * init_cache_level must setup the cache level correctly + * overriding the architecturally specified levels, so + * if type is NONE at this stage, it should be unified + */ + if (this_leaf->type == CACHE_TYPE_NOCACHE && + cache_node_is_unified(this_leaf, np)) + this_leaf->type = CACHE_TYPE_UNIFIED; + cache_size(this_leaf, np); + cache_get_line_size(this_leaf, np); + cache_nr_sets(this_leaf, np); + cache_associativity(this_leaf); + cache_of_set_id(this_leaf, np); +} + +static int cache_setup_of_node(unsigned int cpu) +{ + struct cacheinfo *this_leaf; + unsigned int index = 0; + + struct device_node *np __free(device_node) = of_cpu_device_node_get(cpu); + if (!np) { + pr_err("Failed to find cpu%d device node\n", cpu); + return -ENOENT; + } + + if (!of_check_cache_nodes(np)) { + return -ENOENT; + } + + while (index < cache_leaves(cpu)) { + this_leaf = per_cpu_cacheinfo_idx(cpu, index); + if (this_leaf->level != 1) { + struct device_node *prev __free(device_node) = np; + np = of_find_next_cache_node(np); + if (!np) + break; + } + cache_of_set_props(this_leaf, np); + this_leaf->fw_token = np; + index++; + } + + if (index != cache_leaves(cpu)) /* not all OF nodes populated */ + return -ENOENT; + + return 0; +} + +static bool of_check_cache_nodes(struct device_node *np) +{ + if (of_property_present(np, "cache-size") || + of_property_present(np, "i-cache-size") || + of_property_present(np, "d-cache-size") || + of_property_present(np, "cache-unified")) + return true; + + struct device_node *next __free(device_node) = of_find_next_cache_node(np); + if (next) { + return true; + } + + return false; +} + +static int of_count_cache_leaves(struct device_node *np) +{ + unsigned int leaves = 0; + + if (of_property_present(np, "cache-size")) + ++leaves; + if (of_property_present(np, "i-cache-size")) + ++leaves; + if (of_property_present(np, "d-cache-size")) + ++leaves; + + if (!leaves) { + /* The '[i-|d-|]cache-size' property is required, but + * if absent, fallback on the 'cache-unified' property. + */ + if (of_property_read_bool(np, "cache-unified")) + return 1; + else + return 2; + } + + return leaves; +} + +int init_of_cache_level(unsigned int cpu) +{ + struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); + struct device_node *np __free(device_node) = of_cpu_device_node_get(cpu); + unsigned int levels = 0, leaves, level; + + if (!of_check_cache_nodes(np)) { + return -ENOENT; + } + + leaves = of_count_cache_leaves(np); + if (leaves > 0) + levels = 1; + + while (1) { + struct device_node *prev __free(device_node) = np; + np = of_find_next_cache_node(np); + if (!np) + break; + + if (!of_device_is_compatible(np, "cache")) + return -EINVAL; + if (of_property_read_u32(np, "cache-level", &level)) + return -EINVAL; + if (level <= levels) + return -EINVAL; + + leaves += of_count_cache_leaves(np); + levels = level; + } + + this_cpu_ci->num_levels = levels; + this_cpu_ci->num_leaves = leaves; + + return 0; +} + +#else +static inline int cache_setup_of_node(unsigned int cpu) { return 0; } +int init_of_cache_level(unsigned int cpu) { return 0; } +#endif + +int __weak cache_setup_acpi(unsigned int cpu) +{ + return -ENOTSUPP; +} + +unsigned int coherency_max_size; + +static int cache_setup_properties(unsigned int cpu) +{ + int ret = 0; + + if (of_have_populated_dt()) + ret = cache_setup_of_node(cpu); + else if (!acpi_disabled) + ret = cache_setup_acpi(cpu); + + // Assume there is no cache information available in DT/ACPI from now. + if (ret && use_arch_cache_info()) + use_arch_info = true; + + return ret; +} + +static int cache_shared_cpu_map_setup(unsigned int cpu) +{ + struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); + struct cacheinfo *this_leaf, *sib_leaf; + unsigned int index, sib_index; + int ret = 0; + + if (this_cpu_ci->cpu_map_populated) + return 0; + + /* + * skip setting up cache properties if LLC is valid, just need + * to update the shared cpu_map if the cache attributes were + * populated early before all the cpus are brought online + */ + if (!last_level_cache_is_valid(cpu) && !use_arch_info) { + ret = cache_setup_properties(cpu); + if (ret) + return ret; + } + + for (index = 0; index < cache_leaves(cpu); index++) { + unsigned int i; + + this_leaf = per_cpu_cacheinfo_idx(cpu, index); + + cpumask_set_cpu(cpu, &this_leaf->shared_cpu_map); + for_each_online_cpu(i) { + if (i == cpu || !per_cpu_cacheinfo(i)) + continue;/* skip if itself or no cacheinfo */ + for (sib_index = 0; sib_index < cache_leaves(i); sib_index++) { + sib_leaf = per_cpu_cacheinfo_idx(i, sib_index); + + /* + * Comparing cache IDs only makes sense if the leaves + * belong to the same cache level of same type. Skip + * the check if level and type do not match. + */ + if (sib_leaf->level != this_leaf->level || + sib_leaf->type != this_leaf->type) + continue; + + if (cache_leaves_are_shared(this_leaf, sib_leaf)) { + cpumask_set_cpu(cpu, &sib_leaf->shared_cpu_map); + cpumask_set_cpu(i, &this_leaf->shared_cpu_map); + break; + } + } + } + /* record the maximum cache line size */ + if (this_leaf->coherency_line_size > coherency_max_size) + coherency_max_size = this_leaf->coherency_line_size; + } + + /* shared_cpu_map is now populated for the cpu */ + this_cpu_ci->cpu_map_populated = true; + return 0; +} + +static void cache_shared_cpu_map_remove(unsigned int cpu) +{ + struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); + struct cacheinfo *this_leaf, *sib_leaf; + unsigned int sibling, index, sib_index; + + for (index = 0; index < cache_leaves(cpu); index++) { + this_leaf = per_cpu_cacheinfo_idx(cpu, index); + for_each_cpu(sibling, &this_leaf->shared_cpu_map) { + if (sibling == cpu || !per_cpu_cacheinfo(sibling)) + continue;/* skip if itself or no cacheinfo */ + + for (sib_index = 0; sib_index < cache_leaves(sibling); sib_index++) { + sib_leaf = per_cpu_cacheinfo_idx(sibling, sib_index); + + /* + * Comparing cache IDs only makes sense if the leaves + * belong to the same cache level of same type. Skip + * the check if level and type do not match. + */ + if (sib_leaf->level != this_leaf->level || + sib_leaf->type != this_leaf->type) + continue; + + if (cache_leaves_are_shared(this_leaf, sib_leaf)) { + cpumask_clear_cpu(cpu, &sib_leaf->shared_cpu_map); + cpumask_clear_cpu(sibling, &this_leaf->shared_cpu_map); + break; + } + } + } + } + + /* cpu is no longer populated in the shared map */ + this_cpu_ci->cpu_map_populated = false; +} + +static void free_cache_attributes(unsigned int cpu) +{ + if (!per_cpu_cacheinfo(cpu)) + return; + + cache_shared_cpu_map_remove(cpu); +} + +int __weak early_cache_level(unsigned int cpu) +{ + return -ENOENT; +} + +int __weak init_cache_level(unsigned int cpu) +{ + return -ENOENT; +} + +int __weak populate_cache_leaves(unsigned int cpu) +{ + return -ENOENT; +} + +static inline int allocate_cache_info(int cpu) +{ + per_cpu_cacheinfo(cpu) = kcalloc(cache_leaves(cpu), sizeof(struct cacheinfo), GFP_ATOMIC); + if (!per_cpu_cacheinfo(cpu)) { + cache_leaves(cpu) = 0; + return -ENOMEM; + } + + return 0; +} + +int fetch_cache_info(unsigned int cpu) +{ + struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); + unsigned int levels = 0, split_levels = 0; + int ret; + + if (acpi_disabled) { + ret = init_of_cache_level(cpu); + } else { + ret = acpi_get_cache_info(cpu, &levels, &split_levels); + if (!ret) { + this_cpu_ci->num_levels = levels; + /* + * This assumes that: + * - there cannot be any split caches (data/instruction) + * above a unified cache + * - data/instruction caches come by pair + */ + this_cpu_ci->num_leaves = levels + split_levels; + } + } + + if (ret || !cache_leaves(cpu)) { + ret = early_cache_level(cpu); + if (ret) + return ret; + + if (!cache_leaves(cpu)) + return -ENOENT; + + this_cpu_ci->early_ci_levels = true; + } + + return allocate_cache_info(cpu); +} + +static inline int init_level_allocate_ci(unsigned int cpu) +{ + unsigned int early_leaves = cache_leaves(cpu); + + /* Since early initialization/allocation of the cacheinfo is allowed + * via fetch_cache_info() and this also gets called as CPU hotplug + * callbacks via cacheinfo_cpu_online, the init/alloc can be skipped + * as it will happen only once (the cacheinfo memory is never freed). + * Just populate the cacheinfo. However, if the cacheinfo has been + * allocated early through the arch-specific early_cache_level() call, + * there is a chance the info is wrong (this can happen on arm64). In + * that case, call init_cache_level() anyway to give the arch-specific + * code a chance to make things right. + */ + if (per_cpu_cacheinfo(cpu) && !ci_cacheinfo(cpu)->early_ci_levels) + return 0; + + if (init_cache_level(cpu) || !cache_leaves(cpu)) + return -ENOENT; + + /* + * Now that we have properly initialized the cache level info, make + * sure we don't try to do that again the next time we are called + * (e.g. as CPU hotplug callbacks). + */ + ci_cacheinfo(cpu)->early_ci_levels = false; + + /* + * Some architectures (e.g., x86) do not use early initialization. + * Allocate memory now in such case. + */ + if (cache_leaves(cpu) <= early_leaves && per_cpu_cacheinfo(cpu)) + return 0; + + kfree(per_cpu_cacheinfo(cpu)); + return allocate_cache_info(cpu); +} + +int detect_cache_attributes(unsigned int cpu) +{ + int ret; + + ret = init_level_allocate_ci(cpu); + if (ret) + return ret; + + /* + * If LLC is valid the cache leaves were already populated so just go to + * update the cpu map. + */ + if (!last_level_cache_is_valid(cpu)) { + /* + * populate_cache_leaves() may completely setup the cache leaves and + * shared_cpu_map or it may leave it partially setup. + */ + ret = populate_cache_leaves(cpu); + if (ret) + goto free_ci; + } + + /* + * For systems using DT for cache hierarchy, fw_token + * and shared_cpu_map will be set up here only if they are + * not populated already + */ + ret = cache_shared_cpu_map_setup(cpu); + if (ret) { + pr_warn("Unable to detect cache hierarchy for CPU %d\n", cpu); + goto free_ci; + } + + return 0; + +free_ci: + free_cache_attributes(cpu); + return ret; +} + +/* pointer to cpuX/cache device */ +static DEFINE_PER_CPU(struct device *, ci_cache_dev); +#define per_cpu_cache_dev(cpu) (per_cpu(ci_cache_dev, cpu)) + +static cpumask_t cache_dev_map; + +/* pointer to array of devices for cpuX/cache/indexY */ +static DEFINE_PER_CPU(struct device **, ci_index_dev); +#define per_cpu_index_dev(cpu) (per_cpu(ci_index_dev, cpu)) +#define per_cache_index_dev(cpu, idx) ((per_cpu_index_dev(cpu))[idx]) + +#define show_one(file_name, object) \ +static ssize_t file_name##_show(struct device *dev, \ + struct device_attribute *attr, char *buf) \ +{ \ + struct cacheinfo *this_leaf = dev_get_drvdata(dev); \ + return sysfs_emit(buf, "%u\n", this_leaf->object); \ +} + +show_one(id, id); +show_one(level, level); +show_one(coherency_line_size, coherency_line_size); +show_one(number_of_sets, number_of_sets); +show_one(physical_line_partition, physical_line_partition); +show_one(ways_of_associativity, ways_of_associativity); + +static ssize_t size_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct cacheinfo *this_leaf = dev_get_drvdata(dev); + + return sysfs_emit(buf, "%uK\n", this_leaf->size >> 10); +} + +static ssize_t shared_cpu_map_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct cacheinfo *this_leaf = dev_get_drvdata(dev); + const struct cpumask *mask = &this_leaf->shared_cpu_map; + + return sysfs_emit(buf, "%*pb\n", nr_cpu_ids, mask); +} + +static ssize_t shared_cpu_list_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct cacheinfo *this_leaf = dev_get_drvdata(dev); + const struct cpumask *mask = &this_leaf->shared_cpu_map; + + return sysfs_emit(buf, "%*pbl\n", nr_cpu_ids, mask); +} + +static ssize_t type_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct cacheinfo *this_leaf = dev_get_drvdata(dev); + const char *output; + + switch (this_leaf->type) { + case CACHE_TYPE_DATA: + output = "Data"; + break; + case CACHE_TYPE_INST: + output = "Instruction"; + break; + case CACHE_TYPE_UNIFIED: + output = "Unified"; + break; + default: + return -EINVAL; + } + + return sysfs_emit(buf, "%s\n", output); +} + +static ssize_t allocation_policy_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct cacheinfo *this_leaf = dev_get_drvdata(dev); + unsigned int ci_attr = this_leaf->attributes; + const char *output; + + if ((ci_attr & CACHE_READ_ALLOCATE) && (ci_attr & CACHE_WRITE_ALLOCATE)) + output = "ReadWriteAllocate"; + else if (ci_attr & CACHE_READ_ALLOCATE) + output = "ReadAllocate"; + else if (ci_attr & CACHE_WRITE_ALLOCATE) + output = "WriteAllocate"; + else + return 0; + + return sysfs_emit(buf, "%s\n", output); +} + +static ssize_t write_policy_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct cacheinfo *this_leaf = dev_get_drvdata(dev); + unsigned int ci_attr = this_leaf->attributes; + int n = 0; + + if (ci_attr & CACHE_WRITE_THROUGH) + n = sysfs_emit(buf, "WriteThrough\n"); + else if (ci_attr & CACHE_WRITE_BACK) + n = sysfs_emit(buf, "WriteBack\n"); + return n; +} + +static DEVICE_ATTR_RO(id); +static DEVICE_ATTR_RO(level); +static DEVICE_ATTR_RO(type); +static DEVICE_ATTR_RO(coherency_line_size); +static DEVICE_ATTR_RO(ways_of_associativity); +static DEVICE_ATTR_RO(number_of_sets); +static DEVICE_ATTR_RO(size); +static DEVICE_ATTR_RO(allocation_policy); +static DEVICE_ATTR_RO(write_policy); +static DEVICE_ATTR_RO(shared_cpu_map); +static DEVICE_ATTR_RO(shared_cpu_list); +static DEVICE_ATTR_RO(physical_line_partition); + +static struct attribute *cache_default_attrs[] = { + &dev_attr_id.attr, + &dev_attr_type.attr, + &dev_attr_level.attr, + &dev_attr_shared_cpu_map.attr, + &dev_attr_shared_cpu_list.attr, + &dev_attr_coherency_line_size.attr, + &dev_attr_ways_of_associativity.attr, + &dev_attr_number_of_sets.attr, + &dev_attr_size.attr, + &dev_attr_allocation_policy.attr, + &dev_attr_write_policy.attr, + &dev_attr_physical_line_partition.attr, + NULL +}; + +static umode_t +cache_default_attrs_is_visible(struct kobject *kobj, + struct attribute *attr, int unused) +{ + struct device *dev = kobj_to_dev(kobj); + struct cacheinfo *this_leaf = dev_get_drvdata(dev); + const struct cpumask *mask = &this_leaf->shared_cpu_map; + umode_t mode = attr->mode; + + if ((attr == &dev_attr_id.attr) && (this_leaf->attributes & CACHE_ID)) + return mode; + if ((attr == &dev_attr_type.attr) && this_leaf->type) + return mode; + if ((attr == &dev_attr_level.attr) && this_leaf->level) + return mode; + if ((attr == &dev_attr_shared_cpu_map.attr) && !cpumask_empty(mask)) + return mode; + if ((attr == &dev_attr_shared_cpu_list.attr) && !cpumask_empty(mask)) + return mode; + if ((attr == &dev_attr_coherency_line_size.attr) && + this_leaf->coherency_line_size) + return mode; + if ((attr == &dev_attr_ways_of_associativity.attr) && + this_leaf->size) /* allow 0 = full associativity */ + return mode; + if ((attr == &dev_attr_number_of_sets.attr) && + this_leaf->number_of_sets) + return mode; + if ((attr == &dev_attr_size.attr) && this_leaf->size) + return mode; + if ((attr == &dev_attr_write_policy.attr) && + (this_leaf->attributes & CACHE_WRITE_POLICY_MASK)) + return mode; + if ((attr == &dev_attr_allocation_policy.attr) && + (this_leaf->attributes & CACHE_ALLOCATE_POLICY_MASK)) + return mode; + if ((attr == &dev_attr_physical_line_partition.attr) && + this_leaf->physical_line_partition) + return mode; + + return 0; +} + +static const struct attribute_group cache_default_group = { + .attrs = cache_default_attrs, + .is_visible = cache_default_attrs_is_visible, +}; + +static const struct attribute_group *cache_default_groups[] = { + &cache_default_group, + NULL, +}; + +static const struct attribute_group *cache_private_groups[] = { + &cache_default_group, + NULL, /* Place holder for private group */ + NULL, +}; + +const struct attribute_group * +__weak cache_get_priv_group(struct cacheinfo *this_leaf) +{ + return NULL; +} + +static const struct attribute_group ** +cache_get_attribute_groups(struct cacheinfo *this_leaf) +{ + const struct attribute_group *priv_group = + cache_get_priv_group(this_leaf); + + if (!priv_group) + return cache_default_groups; + + if (!cache_private_groups[1]) + cache_private_groups[1] = priv_group; + + return cache_private_groups; +} + +/* Add/Remove cache interface for CPU device */ +static void cpu_cache_sysfs_exit(unsigned int cpu) +{ + int i; + struct device *ci_dev; + + if (per_cpu_index_dev(cpu)) { + for (i = 0; i < cache_leaves(cpu); i++) { + ci_dev = per_cache_index_dev(cpu, i); + if (!ci_dev) + continue; + device_unregister(ci_dev); + } + kfree(per_cpu_index_dev(cpu)); + per_cpu_index_dev(cpu) = NULL; + } + device_unregister(per_cpu_cache_dev(cpu)); + per_cpu_cache_dev(cpu) = NULL; +} + +static int cpu_cache_sysfs_init(unsigned int cpu) +{ + struct device *dev = get_cpu_device(cpu); + + if (per_cpu_cacheinfo(cpu) == NULL) + return -ENOENT; + + per_cpu_cache_dev(cpu) = cpu_device_create(dev, NULL, NULL, "cache"); + if (IS_ERR(per_cpu_cache_dev(cpu))) + return PTR_ERR(per_cpu_cache_dev(cpu)); + + /* Allocate all required memory */ + per_cpu_index_dev(cpu) = kcalloc(cache_leaves(cpu), + sizeof(struct device *), GFP_KERNEL); + if (unlikely(per_cpu_index_dev(cpu) == NULL)) + goto err_out; + + return 0; + +err_out: + cpu_cache_sysfs_exit(cpu); + return -ENOMEM; +} + +static int cache_add_dev(unsigned int cpu) +{ + unsigned int i; + int rc; + struct device *ci_dev, *parent; + struct cacheinfo *this_leaf; + const struct attribute_group **cache_groups; + + rc = cpu_cache_sysfs_init(cpu); + if (unlikely(rc < 0)) + return rc; + + parent = per_cpu_cache_dev(cpu); + for (i = 0; i < cache_leaves(cpu); i++) { + this_leaf = per_cpu_cacheinfo_idx(cpu, i); + if (this_leaf->disable_sysfs) + continue; + if (this_leaf->type == CACHE_TYPE_NOCACHE) + break; + cache_groups = cache_get_attribute_groups(this_leaf); + ci_dev = cpu_device_create(parent, this_leaf, cache_groups, + "index%1u", i); + if (IS_ERR(ci_dev)) { + rc = PTR_ERR(ci_dev); + goto err; + } + per_cache_index_dev(cpu, i) = ci_dev; + } + cpumask_set_cpu(cpu, &cache_dev_map); + + return 0; +err: + cpu_cache_sysfs_exit(cpu); + return rc; +} + +static unsigned int cpu_map_shared_cache(bool online, unsigned int cpu, + cpumask_t **map) +{ + struct cacheinfo *llc, *sib_llc; + unsigned int sibling; + + if (!last_level_cache_is_valid(cpu)) + return 0; + + llc = per_cpu_cacheinfo_idx(cpu, cache_leaves(cpu) - 1); + + if (llc->type != CACHE_TYPE_DATA && llc->type != CACHE_TYPE_UNIFIED) + return 0; + + if (online) { + *map = &llc->shared_cpu_map; + return cpumask_weight(*map); + } + + /* shared_cpu_map of offlined CPU will be cleared, so use sibling map */ + for_each_cpu(sibling, &llc->shared_cpu_map) { + if (sibling == cpu || !last_level_cache_is_valid(sibling)) + continue; + sib_llc = per_cpu_cacheinfo_idx(sibling, cache_leaves(sibling) - 1); + *map = &sib_llc->shared_cpu_map; + return cpumask_weight(*map); + } + + return 0; +} + +/* + * Calculate the size of the per-CPU data cache slice. This can be + * used to estimate the size of the data cache slice that can be used + * by one CPU under ideal circumstances. UNIFIED caches are counted + * in addition to DATA caches. So, please consider code cache usage + * when use the result. + * + * Because the cache inclusive/non-inclusive information isn't + * available, we just use the size of the per-CPU slice of LLC to make + * the result more predictable across architectures. + */ +static void update_per_cpu_data_slice_size_cpu(unsigned int cpu) +{ + struct cpu_cacheinfo *ci; + struct cacheinfo *llc; + unsigned int nr_shared; + + if (!last_level_cache_is_valid(cpu)) + return; + + ci = ci_cacheinfo(cpu); + llc = per_cpu_cacheinfo_idx(cpu, cache_leaves(cpu) - 1); + + if (llc->type != CACHE_TYPE_DATA && llc->type != CACHE_TYPE_UNIFIED) + return; + + nr_shared = cpumask_weight(&llc->shared_cpu_map); + if (nr_shared) + ci->per_cpu_data_slice_size = llc->size / nr_shared; +} + +static void update_per_cpu_data_slice_size(bool cpu_online, unsigned int cpu, + cpumask_t *cpu_map) +{ + unsigned int icpu; + + for_each_cpu(icpu, cpu_map) { + if (!cpu_online && icpu == cpu) + continue; + update_per_cpu_data_slice_size_cpu(icpu); + setup_pcp_cacheinfo(icpu); + } +} + +static int cacheinfo_cpu_online(unsigned int cpu) +{ + int rc = detect_cache_attributes(cpu); + cpumask_t *cpu_map; + + if (rc) + return rc; + rc = cache_add_dev(cpu); + if (rc) + goto err; + if (cpu_map_shared_cache(true, cpu, &cpu_map)) + update_per_cpu_data_slice_size(true, cpu, cpu_map); + return 0; +err: + free_cache_attributes(cpu); + return rc; +} + +static int cacheinfo_cpu_pre_down(unsigned int cpu) +{ + cpumask_t *cpu_map; + unsigned int nr_shared; + + nr_shared = cpu_map_shared_cache(false, cpu, &cpu_map); + if (cpumask_test_and_clear_cpu(cpu, &cache_dev_map)) + cpu_cache_sysfs_exit(cpu); + + free_cache_attributes(cpu); + if (nr_shared > 1) + update_per_cpu_data_slice_size(false, cpu, cpu_map); + return 0; +} + +static int __init cacheinfo_sysfs_init(void) +{ + return cpuhp_setup_state(CPUHP_AP_BASE_CACHEINFO_ONLINE, + "base/cacheinfo:online", + cacheinfo_cpu_online, cacheinfo_cpu_pre_down); +} +device_initcall(cacheinfo_sysfs_init); diff --git a/drivers/base/class.c b/drivers/base/class.c index 3ce845471327..2526c57d924e 100644 --- a/drivers/base/class.c +++ b/drivers/base/class.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * class.c - basic device class management * @@ -5,11 +6,9 @@ * Copyright (c) 2002-3 Open Source Development Labs * Copyright (c) 2003-2004 Greg Kroah-Hartman * Copyright (c) 2003-2004 IBM Corp. - * - * This file is released under the GPLv2 - * */ +#include <linux/device/class.h> #include <linux/device.h> #include <linux/module.h> #include <linux/init.h> @@ -17,12 +16,56 @@ #include <linux/kdev_t.h> #include <linux/err.h> #include <linux/slab.h> -#include <linux/genhd.h> +#include <linux/blkdev.h> #include <linux/mutex.h> #include "base.h" +/* /sys/class */ +static struct kset *class_kset; + #define to_class_attr(_attr) container_of(_attr, struct class_attribute, attr) +/** + * class_to_subsys - Turn a struct class into a struct subsys_private + * + * @class: pointer to the struct bus_type to look up + * + * The driver core internals need to work on the subsys_private structure, not + * the external struct class pointer. This function walks the list of + * registered classes in the system and finds the matching one and returns the + * internal struct subsys_private that relates to that class. + * + * Note, the reference count of the return value is INCREMENTED if it is not + * NULL. A call to subsys_put() must be done when finished with the pointer in + * order for it to be properly freed. + */ +struct subsys_private *class_to_subsys(const struct class *class) +{ + struct subsys_private *sp = NULL; + struct kobject *kobj; + + if (!class || !class_kset) + return NULL; + + spin_lock(&class_kset->list_lock); + + if (list_empty(&class_kset->list)) + goto done; + + list_for_each_entry(kobj, &class_kset->list, entry) { + struct kset *kset = container_of(kobj, struct kset, kobj); + + sp = container_of_const(kset, struct subsys_private, subsys); + if (sp->class == class) + goto done; + } + sp = NULL; +done: + sp = subsys_get(sp); + spin_unlock(&class_kset->list_lock); + return sp; +} + static ssize_t class_attr_show(struct kobject *kobj, struct attribute *attr, char *buf) { @@ -47,22 +90,10 @@ static ssize_t class_attr_store(struct kobject *kobj, struct attribute *attr, return ret; } -static const void *class_attr_namespace(struct kobject *kobj, - const struct attribute *attr) -{ - struct class_attribute *class_attr = to_class_attr(attr); - struct subsys_private *cp = to_subsys_private(kobj); - const void *ns = NULL; - - if (class_attr->namespace) - ns = class_attr->namespace(cp->class, class_attr); - return ns; -} - static void class_release(struct kobject *kobj) { struct subsys_private *cp = to_subsys_private(kobj); - struct class *class = cp->class; + const struct class *class = cp->class; pr_debug("class '%s': release.\n", class->name); @@ -72,13 +103,14 @@ static void class_release(struct kobject *kobj) pr_debug("class '%s' does not have a release() function, " "be careful\n", class->name); + lockdep_unregister_key(&cp->lock_key); kfree(cp); } -static const struct kobj_ns_type_operations *class_child_ns_type(struct kobject *kobj) +static const struct kobj_ns_type_operations *class_child_ns_type(const struct kobject *kobj) { - struct subsys_private *cp = to_subsys_private(kobj); - struct class *class = cp->class; + const struct subsys_private *cp = to_subsys_private(kobj); + const struct class *class = cp->class; return class->ns_type; } @@ -86,147 +118,134 @@ static const struct kobj_ns_type_operations *class_child_ns_type(struct kobject static const struct sysfs_ops class_sysfs_ops = { .show = class_attr_show, .store = class_attr_store, - .namespace = class_attr_namespace, }; -static struct kobj_type class_ktype = { +static const struct kobj_type class_ktype = { .sysfs_ops = &class_sysfs_ops, .release = class_release, .child_ns_type = class_child_ns_type, }; -/* Hotplug events for classes go to the class subsys */ -static struct kset *class_kset; - - -int class_create_file(struct class *cls, const struct class_attribute *attr) +int class_create_file_ns(const struct class *cls, const struct class_attribute *attr, + const void *ns) { + struct subsys_private *sp = class_to_subsys(cls); int error; - if (cls) - error = sysfs_create_file(&cls->p->subsys.kobj, - &attr->attr); - else - error = -EINVAL; - return error; -} -void class_remove_file(struct class *cls, const struct class_attribute *attr) -{ - if (cls) - sysfs_remove_file(&cls->p->subsys.kobj, &attr->attr); -} + if (!sp) + return -EINVAL; -static struct class *class_get(struct class *cls) -{ - if (cls) - kset_get(&cls->p->subsys); - return cls; -} + error = sysfs_create_file_ns(&sp->subsys.kobj, &attr->attr, ns); + subsys_put(sp); -static void class_put(struct class *cls) -{ - if (cls) - kset_put(&cls->p->subsys); + return error; } +EXPORT_SYMBOL_GPL(class_create_file_ns); -static int add_class_attrs(struct class *cls) +void class_remove_file_ns(const struct class *cls, const struct class_attribute *attr, + const void *ns) { - int i; - int error = 0; + struct subsys_private *sp = class_to_subsys(cls); - if (cls->class_attrs) { - for (i = 0; attr_name(cls->class_attrs[i]); i++) { - error = class_create_file(cls, &cls->class_attrs[i]); - if (error) - goto error; - } - } -done: - return error; -error: - while (--i >= 0) - class_remove_file(cls, &cls->class_attrs[i]); - goto done; + if (!sp) + return; + + sysfs_remove_file_ns(&sp->subsys.kobj, &attr->attr, ns); + subsys_put(sp); } +EXPORT_SYMBOL_GPL(class_remove_file_ns); -static void remove_class_attrs(struct class *cls) +static struct device *klist_class_to_dev(struct klist_node *n) { - int i; - - if (cls->class_attrs) { - for (i = 0; attr_name(cls->class_attrs[i]); i++) - class_remove_file(cls, &cls->class_attrs[i]); - } + struct device_private *p = to_device_private_class(n); + return p->device; } static void klist_class_dev_get(struct klist_node *n) { - struct device *dev = container_of(n, struct device, knode_class); + struct device *dev = klist_class_to_dev(n); get_device(dev); } static void klist_class_dev_put(struct klist_node *n) { - struct device *dev = container_of(n, struct device, knode_class); + struct device *dev = klist_class_to_dev(n); put_device(dev); } -int __class_register(struct class *cls, struct lock_class_key *key) +int class_register(const struct class *cls) { struct subsys_private *cp; + struct lock_class_key *key; int error; pr_debug("device class '%s': registering\n", cls->name); + if (cls->ns_type && !cls->namespace) { + pr_err("%s: class '%s' does not have namespace\n", + __func__, cls->name); + return -EINVAL; + } + if (!cls->ns_type && cls->namespace) { + pr_err("%s: class '%s' does not have ns_type\n", + __func__, cls->name); + return -EINVAL; + } + cp = kzalloc(sizeof(*cp), GFP_KERNEL); if (!cp) return -ENOMEM; klist_init(&cp->klist_devices, klist_class_dev_get, klist_class_dev_put); INIT_LIST_HEAD(&cp->interfaces); kset_init(&cp->glue_dirs); + key = &cp->lock_key; + lockdep_register_key(key); __mutex_init(&cp->mutex, "subsys mutex", key); error = kobject_set_name(&cp->subsys.kobj, "%s", cls->name); - if (error) { - kfree(cp); - return error; - } - - /* set the default /sys/dev directory for devices of this class */ - if (!cls->dev_kobj) - cls->dev_kobj = sysfs_dev_char_kobj; + if (error) + goto err_out; -#if defined(CONFIG_BLOCK) - /* let the block class directory show up in the root of sysfs */ - if (!sysfs_deprecated || cls != &block_class) - cp->subsys.kobj.kset = class_kset; -#else cp->subsys.kobj.kset = class_kset; -#endif cp->subsys.kobj.ktype = &class_ktype; cp->class = cls; - cls->p = cp; error = kset_register(&cp->subsys); + if (error) + goto err_out; + + error = sysfs_create_groups(&cp->subsys.kobj, cls->class_groups); if (error) { - kfree(cp); - return error; + kobject_del(&cp->subsys.kobj); + kfree_const(cp->subsys.kobj.name); + goto err_out; } - error = add_class_attrs(class_get(cls)); - class_put(cls); + return 0; + +err_out: + lockdep_unregister_key(key); + kfree(cp); return error; } -EXPORT_SYMBOL_GPL(__class_register); +EXPORT_SYMBOL_GPL(class_register); -void class_unregister(struct class *cls) +void class_unregister(const struct class *cls) { + struct subsys_private *sp = class_to_subsys(cls); + + if (!sp) + return; + pr_debug("device class '%s': unregistering\n", cls->name); - remove_class_attrs(cls); - kset_unregister(&cls->p->subsys); + + sysfs_remove_groups(&sp->subsys.kobj, cls->class_groups); + kset_unregister(&sp->subsys); + subsys_put(sp); } +EXPORT_SYMBOL_GPL(class_unregister); -static void class_create_release(struct class *cls) +static void class_create_release(const struct class *cls) { pr_debug("%s called for %s\n", __func__, cls->name); kfree(cls); @@ -234,9 +253,7 @@ static void class_create_release(struct class *cls) /** * class_create - create a struct class structure - * @owner: pointer to the module that is to "own" this struct class * @name: pointer to a string for the name of this class. - * @key: the lock_class_key for this class; used by mutex lock debugging * * This is used to create a struct class pointer that can then be used * in calls to device_create(). @@ -246,8 +263,7 @@ static void class_create_release(struct class *cls) * Note, the pointer created here is to be destroyed when finished by * making a call to class_destroy(). */ -struct class *__class_create(struct module *owner, const char *name, - struct lock_class_key *key) +struct class *class_create(const char *name) { struct class *cls; int retval; @@ -259,10 +275,9 @@ struct class *__class_create(struct module *owner, const char *name, } cls->name = name; - cls->owner = owner; cls->class_release = class_create_release; - retval = __class_register(cls, key); + retval = class_register(cls); if (retval) goto error; @@ -272,7 +287,7 @@ error: kfree(cls); return ERR_PTR(retval); } -EXPORT_SYMBOL_GPL(__class_create); +EXPORT_SYMBOL_GPL(class_create); /** * class_destroy - destroys a struct class structure @@ -281,13 +296,14 @@ EXPORT_SYMBOL_GPL(__class_create); * Note, the pointer to be destroyed must have been created with a call * to class_create(). */ -void class_destroy(struct class *cls) +void class_destroy(const struct class *cls) { - if ((cls == NULL) || (IS_ERR(cls))) + if (IS_ERR_OR_NULL(cls)) return; class_unregister(cls); } +EXPORT_SYMBOL_GPL(class_destroy); /** * class_dev_iter_init - initialize class device iterator @@ -301,15 +317,24 @@ void class_destroy(struct class *cls) * otherwise if it is NULL, the iteration starts at the beginning of * the list. */ -void class_dev_iter_init(struct class_dev_iter *iter, struct class *class, - struct device *start, const struct device_type *type) +void class_dev_iter_init(struct class_dev_iter *iter, const struct class *class, + const struct device *start, const struct device_type *type) { + struct subsys_private *sp = class_to_subsys(class); struct klist_node *start_knode = NULL; + memset(iter, 0, sizeof(*iter)); + if (!sp) { + pr_crit("%s: class %p was not registered yet\n", + __func__, class); + return; + } + if (start) - start_knode = &start->knode_class; - klist_iter_init_node(&class->p->klist_devices, &iter->ki, start_knode); + start_knode = &start->p->knode_class; + klist_iter_init_node(&sp->klist_devices, &iter->ki, start_knode); iter->type = type; + iter->sp = sp; } EXPORT_SYMBOL_GPL(class_dev_iter_init); @@ -330,11 +355,14 @@ struct device *class_dev_iter_next(struct class_dev_iter *iter) struct klist_node *knode; struct device *dev; + if (!iter->sp) + return NULL; + while (1) { knode = klist_next(&iter->ki); if (!knode) return NULL; - dev = container_of(knode, struct device, knode_class); + dev = klist_class_to_dev(knode); if (!iter->type || iter->type == dev->type) return dev; } @@ -351,6 +379,7 @@ EXPORT_SYMBOL_GPL(class_dev_iter_next); void class_dev_iter_exit(struct class_dev_iter *iter) { klist_iter_exit(&iter->ki); + subsys_put(iter->sp); } EXPORT_SYMBOL_GPL(class_dev_iter_exit); @@ -372,17 +401,18 @@ EXPORT_SYMBOL_GPL(class_dev_iter_exit); * @fn is allowed to do anything including calling back into class * code. There's no locking restriction. */ -int class_for_each_device(struct class *class, struct device *start, - void *data, int (*fn)(struct device *, void *)) +int class_for_each_device(const struct class *class, const struct device *start, + void *data, device_iter_t fn) { + struct subsys_private *sp = class_to_subsys(class); struct class_dev_iter iter; struct device *dev; int error = 0; if (!class) return -EINVAL; - if (!class->p) { - WARN(1, "%s called for class '%s' before it was initialized", + if (!sp) { + WARN(1, "%s called for class '%s' before it was registered", __func__, class->name); return -EINVAL; } @@ -394,6 +424,7 @@ int class_for_each_device(struct class *class, struct device *start, break; } class_dev_iter_exit(&iter); + subsys_put(sp); return error; } @@ -416,20 +447,20 @@ EXPORT_SYMBOL_GPL(class_for_each_device); * * Note, you will need to drop the reference with put_device() after use. * - * @fn is allowed to do anything including calling back into class + * @match is allowed to do anything including calling back into class * code. There's no locking restriction. */ -struct device *class_find_device(struct class *class, struct device *start, - const void *data, - int (*match)(struct device *, const void *)) +struct device *class_find_device(const struct class *class, const struct device *start, + const void *data, device_match_t match) { + struct subsys_private *sp = class_to_subsys(class); struct class_dev_iter iter; struct device *dev; if (!class) return NULL; - if (!class->p) { - WARN(1, "%s called for class '%s' before it was initialized", + if (!sp) { + WARN(1, "%s called for class '%s' before it was registered", __func__, class->name); return NULL; } @@ -442,6 +473,7 @@ struct device *class_find_device(struct class *class, struct device *start, } } class_dev_iter_exit(&iter); + subsys_put(sp); return dev; } @@ -449,58 +481,79 @@ EXPORT_SYMBOL_GPL(class_find_device); int class_interface_register(struct class_interface *class_intf) { - struct class *parent; + struct subsys_private *sp; + const struct class *parent; struct class_dev_iter iter; struct device *dev; if (!class_intf || !class_intf->class) return -ENODEV; - parent = class_get(class_intf->class); - if (!parent) + parent = class_intf->class; + sp = class_to_subsys(parent); + if (!sp) return -EINVAL; - mutex_lock(&parent->p->mutex); - list_add_tail(&class_intf->node, &parent->p->interfaces); + /* + * Reference in sp is now incremented and will be dropped when + * the interface is removed in the call to class_interface_unregister() + */ + + mutex_lock(&sp->mutex); + list_add_tail(&class_intf->node, &sp->interfaces); if (class_intf->add_dev) { class_dev_iter_init(&iter, parent, NULL, NULL); while ((dev = class_dev_iter_next(&iter))) - class_intf->add_dev(dev, class_intf); + class_intf->add_dev(dev); class_dev_iter_exit(&iter); } - mutex_unlock(&parent->p->mutex); + mutex_unlock(&sp->mutex); return 0; } +EXPORT_SYMBOL_GPL(class_interface_register); void class_interface_unregister(struct class_interface *class_intf) { - struct class *parent = class_intf->class; + struct subsys_private *sp; + const struct class *parent = class_intf->class; struct class_dev_iter iter; struct device *dev; if (!parent) return; - mutex_lock(&parent->p->mutex); + sp = class_to_subsys(parent); + if (!sp) + return; + + mutex_lock(&sp->mutex); list_del_init(&class_intf->node); if (class_intf->remove_dev) { class_dev_iter_init(&iter, parent, NULL, NULL); while ((dev = class_dev_iter_next(&iter))) - class_intf->remove_dev(dev, class_intf); + class_intf->remove_dev(dev); class_dev_iter_exit(&iter); } - mutex_unlock(&parent->p->mutex); + mutex_unlock(&sp->mutex); - class_put(parent); + /* + * Decrement the reference count twice, once for the class_to_subsys() + * call in the start of this function, and the second one from the + * reference increment in class_interface_register() + */ + subsys_put(sp); + subsys_put(sp); } +EXPORT_SYMBOL_GPL(class_interface_unregister); -ssize_t show_class_attr_string(struct class *class, - struct class_attribute *attr, char *buf) +ssize_t show_class_attr_string(const struct class *class, + const struct class_attribute *attr, char *buf) { struct class_attribute_string *cs; + cs = container_of(attr, struct class_attribute_string, attr); - return snprintf(buf, PAGE_SIZE, "%s\n", cs->str); + return sysfs_emit(buf, "%s\n", cs->str); } EXPORT_SYMBOL_GPL(show_class_attr_string); @@ -548,30 +601,10 @@ EXPORT_SYMBOL_GPL(class_compat_unregister); * a bus device * @cls: the compatibility class * @dev: the target bus device - * @device_link: an optional device to which a "device" link should be created */ -int class_compat_create_link(struct class_compat *cls, struct device *dev, - struct device *device_link) +int class_compat_create_link(struct class_compat *cls, struct device *dev) { - int error; - - error = sysfs_create_link(cls->kobj, &dev->kobj, dev_name(dev)); - if (error) - return error; - - /* - * Optionally add a "device" link (typically to the parent), as a - * class device would have one and we want to provide as much - * backwards compatibility as possible. - */ - if (device_link) { - error = sysfs_create_link(&dev->kobj, &device_link->kobj, - "device"); - if (error) - sysfs_remove_link(cls->kobj, dev_name(dev)); - } - - return error; + return sysfs_create_link(cls->kobj, &dev->kobj, dev_name(dev)); } EXPORT_SYMBOL_GPL(class_compat_create_link); @@ -580,18 +613,38 @@ EXPORT_SYMBOL_GPL(class_compat_create_link); * a bus device * @cls: the compatibility class * @dev: the target bus device - * @device_link: an optional device to which a "device" link was previously - * created */ -void class_compat_remove_link(struct class_compat *cls, struct device *dev, - struct device *device_link) +void class_compat_remove_link(struct class_compat *cls, struct device *dev) { - if (device_link) - sysfs_remove_link(&dev->kobj, "device"); sysfs_remove_link(cls->kobj, dev_name(dev)); } EXPORT_SYMBOL_GPL(class_compat_remove_link); +/** + * class_is_registered - determine if at this moment in time, a class is + * registered in the driver core or not. + * @class: the class to check + * + * Returns a boolean to state if the class is registered in the driver core + * or not. Note that the value could switch right after this call is made, + * so only use this in places where you "know" it is safe to do so (usually + * to determine if the specific class has been registered yet or not). + * + * Be careful in using this. + */ +bool class_is_registered(const struct class *class) +{ + struct subsys_private *sp = class_to_subsys(class); + bool is_initialized = false; + + if (sp) { + is_initialized = true; + subsys_put(sp); + } + return is_initialized; +} +EXPORT_SYMBOL_GPL(class_is_registered); + int __init classes_init(void) { class_kset = kset_create_and_add("class", NULL, NULL); @@ -599,11 +652,3 @@ int __init classes_init(void) return -ENOMEM; return 0; } - -EXPORT_SYMBOL_GPL(class_create_file); -EXPORT_SYMBOL_GPL(class_remove_file); -EXPORT_SYMBOL_GPL(class_unregister); -EXPORT_SYMBOL_GPL(class_destroy); - -EXPORT_SYMBOL_GPL(class_interface_register); -EXPORT_SYMBOL_GPL(class_interface_unregister); diff --git a/drivers/base/component.c b/drivers/base/component.c new file mode 100644 index 000000000000..024ad9471b8a --- /dev/null +++ b/drivers/base/component.c @@ -0,0 +1,842 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Componentized device handling. + */ +#include <linux/component.h> +#include <linux/device.h> +#include <linux/list.h> +#include <linux/mutex.h> +#include <linux/of.h> +#include <linux/slab.h> +#include <linux/debugfs.h> + +/** + * DOC: overview + * + * The component helper allows drivers to collect a pile of sub-devices, + * including their bound drivers, into an aggregate driver. Various subsystems + * already provide functions to get hold of such components, e.g. + * of_clk_get_by_name(). The component helper can be used when such a + * subsystem-specific way to find a device is not available: The component + * helper fills the niche of aggregate drivers for specific hardware, where + * further standardization into a subsystem would not be practical. The common + * example is when a logical device (e.g. a DRM display driver) is spread around + * the SoC on various components (scanout engines, blending blocks, transcoders + * for various outputs and so on). + * + * The component helper also doesn't solve runtime dependencies, e.g. for system + * suspend and resume operations. See also :ref:`device links<device_link>`. + * + * Components are registered using component_add() and unregistered with + * component_del(), usually from the driver's probe and disconnect functions. + * + * Aggregate drivers first assemble a component match list of what they need + * using component_match_add(). This is then registered as an aggregate driver + * using component_master_add_with_match(), and unregistered using + * component_master_del(). + */ + +struct component; + +struct component_match_array { + void *data; + int (*compare)(struct device *, void *); + int (*compare_typed)(struct device *, int, void *); + void (*release)(struct device *, void *); + struct component *component; + bool duplicate; +}; + +struct component_match { + size_t alloc; + size_t num; + struct component_match_array *compare; +}; + +struct aggregate_device { + struct list_head node; + bool bound; + + const struct component_master_ops *ops; + struct device *parent; + struct component_match *match; +}; + +struct component { + struct list_head node; + struct aggregate_device *adev; + bool bound; + + const struct component_ops *ops; + int subcomponent; + struct device *dev; +}; + +static DEFINE_MUTEX(component_mutex); +static LIST_HEAD(component_list); +static LIST_HEAD(aggregate_devices); + +#ifdef CONFIG_DEBUG_FS + +static struct dentry *component_debugfs_dir; + +static int component_devices_show(struct seq_file *s, void *data) +{ + struct aggregate_device *m = s->private; + struct component_match *match = m->match; + size_t i; + + mutex_lock(&component_mutex); + seq_printf(s, "%-50s %20s\n", "aggregate_device name", "status"); + seq_puts(s, "-----------------------------------------------------------------------\n"); + seq_printf(s, "%-50s %20s\n\n", + dev_name(m->parent), m->bound ? "bound" : "not bound"); + + seq_printf(s, "%-50s %20s\n", "device name", "status"); + seq_puts(s, "-----------------------------------------------------------------------\n"); + for (i = 0; i < match->num; i++) { + struct component *component = match->compare[i].component; + + seq_printf(s, "%-50s %20s\n", + component ? dev_name(component->dev) : "(unknown)", + component ? (component->bound ? "bound" : "not bound") : "not registered"); + } + mutex_unlock(&component_mutex); + + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(component_devices); + +static int __init component_debug_init(void) +{ + component_debugfs_dir = debugfs_create_dir("device_component", NULL); + + return 0; +} + +core_initcall(component_debug_init); + +static void component_debugfs_add(struct aggregate_device *m) +{ + debugfs_create_file(dev_name(m->parent), 0444, component_debugfs_dir, m, + &component_devices_fops); +} + +static void component_debugfs_del(struct aggregate_device *m) +{ + debugfs_lookup_and_remove(dev_name(m->parent), component_debugfs_dir); +} + +#else + +static void component_debugfs_add(struct aggregate_device *m) +{ } + +static void component_debugfs_del(struct aggregate_device *m) +{ } + +#endif + +static struct aggregate_device *__aggregate_find(struct device *parent, + const struct component_master_ops *ops) +{ + struct aggregate_device *m; + + list_for_each_entry(m, &aggregate_devices, node) + if (m->parent == parent && (!ops || m->ops == ops)) + return m; + + return NULL; +} + +static struct component *find_component(struct aggregate_device *adev, + struct component_match_array *mc) +{ + struct component *c; + + list_for_each_entry(c, &component_list, node) { + if (c->adev && c->adev != adev) + continue; + + if (mc->compare && mc->compare(c->dev, mc->data)) + return c; + + if (mc->compare_typed && + mc->compare_typed(c->dev, c->subcomponent, mc->data)) + return c; + } + + return NULL; +} + +static int find_components(struct aggregate_device *adev) +{ + struct component_match *match = adev->match; + size_t i; + int ret = 0; + + /* + * Scan the array of match functions and attach + * any components which are found to this adev. + */ + for (i = 0; i < match->num; i++) { + struct component_match_array *mc = &match->compare[i]; + struct component *c; + + dev_dbg(adev->parent, "Looking for component %zu\n", i); + + if (match->compare[i].component) + continue; + + c = find_component(adev, mc); + if (!c) { + ret = -ENXIO; + break; + } + + dev_dbg(adev->parent, "found component %s, duplicate %u\n", + dev_name(c->dev), !!c->adev); + + /* Attach this component to the adev */ + match->compare[i].duplicate = !!c->adev; + match->compare[i].component = c; + c->adev = adev; + } + return ret; +} + +/* Detach component from associated aggregate_device */ +static void remove_component(struct aggregate_device *adev, struct component *c) +{ + size_t i; + + /* Detach the component from this adev. */ + for (i = 0; i < adev->match->num; i++) + if (adev->match->compare[i].component == c) + adev->match->compare[i].component = NULL; +} + +/* + * Try to bring up an aggregate device. If component is NULL, we're interested + * in this aggregate device, otherwise it's a component which must be present + * to try and bring up the aggregate device. + * + * Returns 1 for successful bringup, 0 if not ready, or -ve errno. + */ +static int try_to_bring_up_aggregate_device(struct aggregate_device *adev, + struct component *component) +{ + int ret; + + dev_dbg(adev->parent, "trying to bring up adev\n"); + + if (find_components(adev)) { + dev_dbg(adev->parent, "master has incomplete components\n"); + return 0; + } + + if (component && component->adev != adev) { + dev_dbg(adev->parent, "master is not for this component (%s)\n", + dev_name(component->dev)); + return 0; + } + + if (!devres_open_group(adev->parent, adev, GFP_KERNEL)) + return -ENOMEM; + + /* Found all components */ + ret = adev->ops->bind(adev->parent); + if (ret < 0) { + devres_release_group(adev->parent, NULL); + if (ret != -EPROBE_DEFER) + dev_info(adev->parent, "adev bind failed: %d\n", ret); + return ret; + } + + devres_close_group(adev->parent, NULL); + adev->bound = true; + return 1; +} + +static int try_to_bring_up_masters(struct component *component) +{ + struct aggregate_device *adev; + int ret = 0; + + list_for_each_entry(adev, &aggregate_devices, node) { + if (!adev->bound) { + ret = try_to_bring_up_aggregate_device(adev, component); + if (ret != 0) + break; + } + } + + return ret; +} + +static void take_down_aggregate_device(struct aggregate_device *adev) +{ + if (adev->bound) { + adev->ops->unbind(adev->parent); + devres_release_group(adev->parent, adev); + adev->bound = false; + } +} + +/** + * component_compare_of - A common component compare function for of_node + * @dev: component device + * @data: @compare_data from component_match_add_release() + * + * A common compare function when compare_data is device of_node. e.g. + * component_match_add_release(masterdev, &match, component_release_of, + * component_compare_of, component_dev_of_node) + */ +int component_compare_of(struct device *dev, void *data) +{ + return device_match_of_node(dev, data); +} +EXPORT_SYMBOL_GPL(component_compare_of); + +/** + * component_release_of - A common component release function for of_node + * @dev: component device + * @data: @compare_data from component_match_add_release() + * + * About the example, Please see component_compare_of(). + */ +void component_release_of(struct device *dev, void *data) +{ + of_node_put(data); +} +EXPORT_SYMBOL_GPL(component_release_of); + +/** + * component_compare_dev - A common component compare function for dev + * @dev: component device + * @data: @compare_data from component_match_add_release() + * + * A common compare function when compare_data is struce device. e.g. + * component_match_add(masterdev, &match, component_compare_dev, component_dev) + */ +int component_compare_dev(struct device *dev, void *data) +{ + return dev == data; +} +EXPORT_SYMBOL_GPL(component_compare_dev); + +/** + * component_compare_dev_name - A common component compare function for device name + * @dev: component device + * @data: @compare_data from component_match_add_release() + * + * A common compare function when compare_data is device name string. e.g. + * component_match_add(masterdev, &match, component_compare_dev_name, + * "component_dev_name") + */ +int component_compare_dev_name(struct device *dev, void *data) +{ + return device_match_name(dev, data); +} +EXPORT_SYMBOL_GPL(component_compare_dev_name); + +static void devm_component_match_release(struct device *parent, void *res) +{ + struct component_match *match = res; + unsigned int i; + + for (i = 0; i < match->num; i++) { + struct component_match_array *mc = &match->compare[i]; + + if (mc->release) + mc->release(parent, mc->data); + } + + kfree(match->compare); +} + +static int component_match_realloc(struct component_match *match, size_t num) +{ + struct component_match_array *new; + + if (match->alloc == num) + return 0; + + new = kmalloc_array(num, sizeof(*new), GFP_KERNEL); + if (!new) + return -ENOMEM; + + if (match->compare) { + memcpy(new, match->compare, sizeof(*new) * + min(match->num, num)); + kfree(match->compare); + } + match->compare = new; + match->alloc = num; + + return 0; +} + +static void __component_match_add(struct device *parent, + struct component_match **matchptr, + void (*release)(struct device *, void *), + int (*compare)(struct device *, void *), + int (*compare_typed)(struct device *, int, void *), + void *compare_data) +{ + struct component_match *match = *matchptr; + + if (IS_ERR(match)) + return; + + if (!match) { + match = devres_alloc(devm_component_match_release, + sizeof(*match), GFP_KERNEL); + if (!match) { + *matchptr = ERR_PTR(-ENOMEM); + return; + } + + devres_add(parent, match); + + *matchptr = match; + } + + if (match->num == match->alloc) { + size_t new_size = match->alloc + 16; + int ret; + + ret = component_match_realloc(match, new_size); + if (ret) { + *matchptr = ERR_PTR(ret); + return; + } + } + + match->compare[match->num].compare = compare; + match->compare[match->num].compare_typed = compare_typed; + match->compare[match->num].release = release; + match->compare[match->num].data = compare_data; + match->compare[match->num].component = NULL; + match->num++; +} + +/** + * component_match_add_release - add a component match entry with release callback + * @parent: parent device of the aggregate driver + * @matchptr: pointer to the list of component matches + * @release: release function for @compare_data + * @compare: compare function to match against all components + * @compare_data: opaque pointer passed to the @compare function + * + * Adds a new component match to the list stored in @matchptr, which the + * aggregate driver needs to function. The list of component matches pointed to + * by @matchptr must be initialized to NULL before adding the first match. This + * only matches against components added with component_add(). + * + * The allocated match list in @matchptr is automatically released using devm + * actions, where upon @release will be called to free any references held by + * @compare_data, e.g. when @compare_data is a &device_node that must be + * released with of_node_put(). + * + * See also component_match_add() and component_match_add_typed(). + */ +void component_match_add_release(struct device *parent, + struct component_match **matchptr, + void (*release)(struct device *, void *), + int (*compare)(struct device *, void *), void *compare_data) +{ + __component_match_add(parent, matchptr, release, compare, NULL, + compare_data); +} +EXPORT_SYMBOL(component_match_add_release); + +/** + * component_match_add_typed - add a component match entry for a typed component + * @parent: parent device of the aggregate driver + * @matchptr: pointer to the list of component matches + * @compare_typed: compare function to match against all typed components + * @compare_data: opaque pointer passed to the @compare function + * + * Adds a new component match to the list stored in @matchptr, which the + * aggregate driver needs to function. The list of component matches pointed to + * by @matchptr must be initialized to NULL before adding the first match. This + * only matches against components added with component_add_typed(). + * + * The allocated match list in @matchptr is automatically released using devm + * actions. + * + * See also component_match_add_release() and component_match_add_typed(). + */ +void component_match_add_typed(struct device *parent, + struct component_match **matchptr, + int (*compare_typed)(struct device *, int, void *), void *compare_data) +{ + __component_match_add(parent, matchptr, NULL, NULL, compare_typed, + compare_data); +} +EXPORT_SYMBOL(component_match_add_typed); + +static void free_aggregate_device(struct aggregate_device *adev) +{ + struct component_match *match = adev->match; + int i; + + component_debugfs_del(adev); + list_del(&adev->node); + + if (match) { + for (i = 0; i < match->num; i++) { + struct component *c = match->compare[i].component; + if (c) + c->adev = NULL; + } + } + + kfree(adev); +} + +/** + * component_master_add_with_match - register an aggregate driver + * @parent: parent device of the aggregate driver + * @ops: callbacks for the aggregate driver + * @match: component match list for the aggregate driver + * + * Registers a new aggregate driver consisting of the components added to @match + * by calling one of the component_match_add() functions. Once all components in + * @match are available, it will be assembled by calling + * &component_master_ops.bind from @ops. Must be unregistered by calling + * component_master_del(). + */ +int component_master_add_with_match(struct device *parent, + const struct component_master_ops *ops, + struct component_match *match) +{ + struct aggregate_device *adev; + int ret; + + /* Reallocate the match array for its true size */ + ret = component_match_realloc(match, match->num); + if (ret) + return ret; + + adev = kzalloc(sizeof(*adev), GFP_KERNEL); + if (!adev) + return -ENOMEM; + + adev->parent = parent; + adev->ops = ops; + adev->match = match; + + component_debugfs_add(adev); + /* Add to the list of available aggregate devices. */ + mutex_lock(&component_mutex); + list_add(&adev->node, &aggregate_devices); + + ret = try_to_bring_up_aggregate_device(adev, NULL); + + if (ret < 0) + free_aggregate_device(adev); + + mutex_unlock(&component_mutex); + + return ret < 0 ? ret : 0; +} +EXPORT_SYMBOL_GPL(component_master_add_with_match); + +/** + * component_master_del - unregister an aggregate driver + * @parent: parent device of the aggregate driver + * @ops: callbacks for the aggregate driver + * + * Unregisters an aggregate driver registered with + * component_master_add_with_match(). If necessary the aggregate driver is first + * disassembled by calling &component_master_ops.unbind from @ops. + */ +void component_master_del(struct device *parent, + const struct component_master_ops *ops) +{ + struct aggregate_device *adev; + + mutex_lock(&component_mutex); + adev = __aggregate_find(parent, ops); + if (adev) { + take_down_aggregate_device(adev); + free_aggregate_device(adev); + } + mutex_unlock(&component_mutex); +} +EXPORT_SYMBOL_GPL(component_master_del); + +bool component_master_is_bound(struct device *parent, + const struct component_master_ops *ops) +{ + struct aggregate_device *adev; + + guard(mutex)(&component_mutex); + adev = __aggregate_find(parent, ops); + if (!adev) + return 0; + + return adev->bound; +} +EXPORT_SYMBOL_GPL(component_master_is_bound); + +static void component_unbind(struct component *component, + struct aggregate_device *adev, void *data) +{ + if (WARN_ON(!component->bound)) + return; + + dev_dbg(adev->parent, "unbinding %s component %p (ops %ps)\n", + dev_name(component->dev), component, component->ops); + + if (component->ops && component->ops->unbind) + component->ops->unbind(component->dev, adev->parent, data); + component->bound = false; + + /* Release all resources claimed in the binding of this component */ + devres_release_group(component->dev, component); +} + +/** + * component_unbind_all - unbind all components of an aggregate driver + * @parent: parent device of the aggregate driver + * @data: opaque pointer, passed to all components + * + * Unbinds all components of the aggregate device by passing @data to their + * &component_ops.unbind functions. Should be called from + * &component_master_ops.unbind. + */ +void component_unbind_all(struct device *parent, void *data) +{ + struct aggregate_device *adev; + struct component *c; + size_t i; + + WARN_ON(!mutex_is_locked(&component_mutex)); + + adev = __aggregate_find(parent, NULL); + if (!adev) + return; + + /* Unbind components in reverse order */ + for (i = adev->match->num; i--; ) + if (!adev->match->compare[i].duplicate) { + c = adev->match->compare[i].component; + component_unbind(c, adev, data); + } +} +EXPORT_SYMBOL_GPL(component_unbind_all); + +static int component_bind(struct component *component, struct aggregate_device *adev, + void *data) +{ + int ret; + + /* + * Each component initialises inside its own devres group. + * This allows us to roll-back a failed component without + * affecting anything else. + */ + if (!devres_open_group(adev->parent, NULL, GFP_KERNEL)) + return -ENOMEM; + + /* + * Also open a group for the device itself: this allows us + * to release the resources claimed against the sub-device + * at the appropriate moment. + */ + if (!devres_open_group(component->dev, component, GFP_KERNEL)) { + devres_release_group(adev->parent, NULL); + return -ENOMEM; + } + + dev_dbg(adev->parent, "binding %s (ops %ps)\n", + dev_name(component->dev), component->ops); + + ret = component->ops->bind(component->dev, adev->parent, data); + if (!ret) { + component->bound = true; + + /* + * Close the component device's group so that resources + * allocated in the binding are encapsulated for removal + * at unbind. Remove the group on the DRM device as we + * can clean those resources up independently. + */ + devres_close_group(component->dev, NULL); + devres_remove_group(adev->parent, NULL); + + dev_info(adev->parent, "bound %s (ops %ps)\n", + dev_name(component->dev), component->ops); + } else { + devres_release_group(component->dev, NULL); + devres_release_group(adev->parent, NULL); + + if (ret != -EPROBE_DEFER) + dev_err(adev->parent, "failed to bind %s (ops %ps): %d\n", + dev_name(component->dev), component->ops, ret); + } + + return ret; +} + +/** + * component_bind_all - bind all components of an aggregate driver + * @parent: parent device of the aggregate driver + * @data: opaque pointer, passed to all components + * + * Binds all components of the aggregate @dev by passing @data to their + * &component_ops.bind functions. Should be called from + * &component_master_ops.bind. + */ +int component_bind_all(struct device *parent, void *data) +{ + struct aggregate_device *adev; + struct component *c; + size_t i; + int ret = 0; + + WARN_ON(!mutex_is_locked(&component_mutex)); + + adev = __aggregate_find(parent, NULL); + if (!adev) + return -EINVAL; + + /* Bind components in match order */ + for (i = 0; i < adev->match->num; i++) + if (!adev->match->compare[i].duplicate) { + c = adev->match->compare[i].component; + ret = component_bind(c, adev, data); + if (ret) + break; + } + + if (ret != 0) { + for (; i > 0; i--) + if (!adev->match->compare[i - 1].duplicate) { + c = adev->match->compare[i - 1].component; + component_unbind(c, adev, data); + } + } + + return ret; +} +EXPORT_SYMBOL_GPL(component_bind_all); + +static int __component_add(struct device *dev, const struct component_ops *ops, + int subcomponent) +{ + struct component *component; + int ret; + + component = kzalloc(sizeof(*component), GFP_KERNEL); + if (!component) + return -ENOMEM; + + component->ops = ops; + component->dev = dev; + component->subcomponent = subcomponent; + + dev_dbg(dev, "adding component (ops %ps)\n", ops); + + mutex_lock(&component_mutex); + list_add_tail(&component->node, &component_list); + + ret = try_to_bring_up_masters(component); + if (ret < 0) { + if (component->adev) + remove_component(component->adev, component); + list_del(&component->node); + + kfree(component); + } + mutex_unlock(&component_mutex); + + return ret < 0 ? ret : 0; +} + +/** + * component_add_typed - register a component + * @dev: component device + * @ops: component callbacks + * @subcomponent: nonzero identifier for subcomponents + * + * Register a new component for @dev. Functions in @ops will be call when the + * aggregate driver is ready to bind the overall driver by calling + * component_bind_all(). See also &struct component_ops. + * + * @subcomponent must be nonzero and is used to differentiate between multiple + * components registered on the same device @dev. These components are match + * using component_match_add_typed(). + * + * The component needs to be unregistered at driver unload/disconnect by + * calling component_del(). + * + * See also component_add(). + */ +int component_add_typed(struct device *dev, const struct component_ops *ops, + int subcomponent) +{ + if (WARN_ON(subcomponent == 0)) + return -EINVAL; + + return __component_add(dev, ops, subcomponent); +} +EXPORT_SYMBOL_GPL(component_add_typed); + +/** + * component_add - register a component + * @dev: component device + * @ops: component callbacks + * + * Register a new component for @dev. Functions in @ops will be called when the + * aggregate driver is ready to bind the overall driver by calling + * component_bind_all(). See also &struct component_ops. + * + * The component needs to be unregistered at driver unload/disconnect by + * calling component_del(). + * + * See also component_add_typed() for a variant that allows multiple different + * components on the same device. + */ +int component_add(struct device *dev, const struct component_ops *ops) +{ + return __component_add(dev, ops, 0); +} +EXPORT_SYMBOL_GPL(component_add); + +/** + * component_del - unregister a component + * @dev: component device + * @ops: component callbacks + * + * Unregister a component added with component_add(). If the component is bound + * into an aggregate driver, this will force the entire aggregate driver, including + * all its components, to be unbound. + */ +void component_del(struct device *dev, const struct component_ops *ops) +{ + struct component *c, *component = NULL; + + mutex_lock(&component_mutex); + list_for_each_entry(c, &component_list, node) + if (c->dev == dev && c->ops == ops) { + list_del(&c->node); + component = c; + break; + } + + if (component && component->adev) { + take_down_aggregate_device(component->adev); + remove_component(component->adev, component); + } + + mutex_unlock(&component_mutex); + + WARN_ON(!component); + kfree(component); +} +EXPORT_SYMBOL_GPL(component_del); diff --git a/drivers/base/container.c b/drivers/base/container.c new file mode 100644 index 000000000000..f40588ebc3f5 --- /dev/null +++ b/drivers/base/container.c @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * System bus type for containers. + * + * Copyright (C) 2013, Intel Corporation + * Author: Rafael J. Wysocki <rafael.j.wysocki@intel.com> + */ + +#include <linux/container.h> + +#include "base.h" + +#define CONTAINER_BUS_NAME "container" + +static int trivial_online(struct device *dev) +{ + return 0; +} + +static int container_offline(struct device *dev) +{ + struct container_dev *cdev = to_container_dev(dev); + + return cdev->offline ? cdev->offline(cdev) : 0; +} + +const struct bus_type container_subsys = { + .name = CONTAINER_BUS_NAME, + .dev_name = CONTAINER_BUS_NAME, + .online = trivial_online, + .offline = container_offline, +}; + +void __init container_dev_init(void) +{ + int ret; + + ret = subsys_system_register(&container_subsys, NULL); + if (ret) + pr_err("%s() failed: %d\n", __func__, ret); +} diff --git a/drivers/base/core.c b/drivers/base/core.c index dc3ea237f086..40de2f51a1b1 100644 --- a/drivers/base/core.c +++ b/drivers/base/core.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * drivers/base/core.c - core driver model code (device registration, etc) * @@ -5,49 +6,2359 @@ * Copyright (c) 2002-3 Open Source Development Labs * Copyright (c) 2006 Greg Kroah-Hartman <gregkh@suse.de> * Copyright (c) 2006 Novell, Inc. - * - * This file is released under the GPLv2 - * */ +#include <linux/acpi.h> +#include <linux/blkdev.h> +#include <linux/cleanup.h> +#include <linux/cpufreq.h> #include <linux/device.h> +#include <linux/dma-map-ops.h> /* for dma_default_coherent */ #include <linux/err.h> +#include <linux/fwnode.h> #include <linux/init.h> -#include <linux/module.h> -#include <linux/slab.h> -#include <linux/string.h> #include <linux/kdev_t.h> +#include <linux/kstrtox.h> +#include <linux/module.h> +#include <linux/mutex.h> +#include <linux/netdevice.h> #include <linux/notifier.h> #include <linux/of.h> #include <linux/of_device.h> -#include <linux/genhd.h> -#include <linux/kallsyms.h> -#include <linux/mutex.h> -#include <linux/async.h> #include <linux/pm_runtime.h> -#include <linux/netdevice.h> +#include <linux/sched/mm.h> +#include <linux/sched/signal.h> +#include <linux/slab.h> +#include <linux/string_helpers.h> +#include <linux/swiotlb.h> +#include <linux/sysfs.h> #include "base.h" +#include "physical_location.h" #include "power/power.h" -#ifdef CONFIG_SYSFS_DEPRECATED -#ifdef CONFIG_SYSFS_DEPRECATED_V2 -long sysfs_deprecated = 1; +/* Device links support. */ +static LIST_HEAD(deferred_sync); +static unsigned int defer_sync_state_count = 1; +static DEFINE_MUTEX(fwnode_link_lock); +static bool fw_devlink_is_permissive(void); +static void __fw_devlink_link_to_consumers(struct device *dev); +static bool fw_devlink_drv_reg_done; +static bool fw_devlink_best_effort; +static struct workqueue_struct *device_link_wq; + +/** + * __fwnode_link_add - Create a link between two fwnode_handles. + * @con: Consumer end of the link. + * @sup: Supplier end of the link. + * @flags: Link flags. + * + * Create a fwnode link between fwnode handles @con and @sup. The fwnode link + * represents the detail that the firmware lists @sup fwnode as supplying a + * resource to @con. + * + * The driver core will use the fwnode link to create a device link between the + * two device objects corresponding to @con and @sup when they are created. The + * driver core will automatically delete the fwnode link between @con and @sup + * after doing that. + * + * Attempts to create duplicate links between the same pair of fwnode handles + * are ignored and there is no reference counting. + */ +static int __fwnode_link_add(struct fwnode_handle *con, + struct fwnode_handle *sup, u8 flags) +{ + struct fwnode_link *link; + + list_for_each_entry(link, &sup->consumers, s_hook) + if (link->consumer == con) { + link->flags |= flags; + return 0; + } + + link = kzalloc(sizeof(*link), GFP_KERNEL); + if (!link) + return -ENOMEM; + + link->supplier = sup; + INIT_LIST_HEAD(&link->s_hook); + link->consumer = con; + INIT_LIST_HEAD(&link->c_hook); + link->flags = flags; + + list_add(&link->s_hook, &sup->consumers); + list_add(&link->c_hook, &con->suppliers); + pr_debug("%pfwf Linked as a fwnode consumer to %pfwf\n", + con, sup); + + return 0; +} + +int fwnode_link_add(struct fwnode_handle *con, struct fwnode_handle *sup, + u8 flags) +{ + guard(mutex)(&fwnode_link_lock); + + return __fwnode_link_add(con, sup, flags); +} + +/** + * __fwnode_link_del - Delete a link between two fwnode_handles. + * @link: the fwnode_link to be deleted + * + * The fwnode_link_lock needs to be held when this function is called. + */ +static void __fwnode_link_del(struct fwnode_link *link) +{ + pr_debug("%pfwf Dropping the fwnode link to %pfwf\n", + link->consumer, link->supplier); + list_del(&link->s_hook); + list_del(&link->c_hook); + kfree(link); +} + +/** + * __fwnode_link_cycle - Mark a fwnode link as being part of a cycle. + * @link: the fwnode_link to be marked + * + * The fwnode_link_lock needs to be held when this function is called. + */ +static void __fwnode_link_cycle(struct fwnode_link *link) +{ + pr_debug("%pfwf: cycle: depends on %pfwf\n", + link->consumer, link->supplier); + link->flags |= FWLINK_FLAG_CYCLE; +} + +/** + * fwnode_links_purge_suppliers - Delete all supplier links of fwnode_handle. + * @fwnode: fwnode whose supplier links need to be deleted + * + * Deletes all supplier links connecting directly to @fwnode. + */ +static void fwnode_links_purge_suppliers(struct fwnode_handle *fwnode) +{ + struct fwnode_link *link, *tmp; + + guard(mutex)(&fwnode_link_lock); + + list_for_each_entry_safe(link, tmp, &fwnode->suppliers, c_hook) + __fwnode_link_del(link); +} + +/** + * fwnode_links_purge_consumers - Delete all consumer links of fwnode_handle. + * @fwnode: fwnode whose consumer links need to be deleted + * + * Deletes all consumer links connecting directly to @fwnode. + */ +static void fwnode_links_purge_consumers(struct fwnode_handle *fwnode) +{ + struct fwnode_link *link, *tmp; + + guard(mutex)(&fwnode_link_lock); + + list_for_each_entry_safe(link, tmp, &fwnode->consumers, s_hook) + __fwnode_link_del(link); +} + +/** + * fwnode_links_purge - Delete all links connected to a fwnode_handle. + * @fwnode: fwnode whose links needs to be deleted + * + * Deletes all links connecting directly to a fwnode. + */ +void fwnode_links_purge(struct fwnode_handle *fwnode) +{ + fwnode_links_purge_suppliers(fwnode); + fwnode_links_purge_consumers(fwnode); +} + +void fw_devlink_purge_absent_suppliers(struct fwnode_handle *fwnode) +{ + struct fwnode_handle *child; + + /* Don't purge consumer links of an added child */ + if (fwnode->dev) + return; + + fwnode->flags |= FWNODE_FLAG_NOT_DEVICE; + fwnode_links_purge_consumers(fwnode); + + fwnode_for_each_available_child_node(fwnode, child) + fw_devlink_purge_absent_suppliers(child); +} +EXPORT_SYMBOL_GPL(fw_devlink_purge_absent_suppliers); + +/** + * __fwnode_links_move_consumers - Move consumer from @from to @to fwnode_handle + * @from: move consumers away from this fwnode + * @to: move consumers to this fwnode + * + * Move all consumer links from @from fwnode to @to fwnode. + */ +static void __fwnode_links_move_consumers(struct fwnode_handle *from, + struct fwnode_handle *to) +{ + struct fwnode_link *link, *tmp; + + list_for_each_entry_safe(link, tmp, &from->consumers, s_hook) { + __fwnode_link_add(link->consumer, to, link->flags); + __fwnode_link_del(link); + } +} + +/** + * __fw_devlink_pickup_dangling_consumers - Pick up dangling consumers + * @fwnode: fwnode from which to pick up dangling consumers + * @new_sup: fwnode of new supplier + * + * If the @fwnode has a corresponding struct device and the device supports + * probing (that is, added to a bus), then we want to let fw_devlink create + * MANAGED device links to this device, so leave @fwnode and its descendant's + * fwnode links alone. + * + * Otherwise, move its consumers to the new supplier @new_sup. + */ +static void __fw_devlink_pickup_dangling_consumers(struct fwnode_handle *fwnode, + struct fwnode_handle *new_sup) +{ + struct fwnode_handle *child; + + if (fwnode->dev && fwnode->dev->bus) + return; + + fwnode->flags |= FWNODE_FLAG_NOT_DEVICE; + __fwnode_links_move_consumers(fwnode, new_sup); + + fwnode_for_each_available_child_node(fwnode, child) + __fw_devlink_pickup_dangling_consumers(child, new_sup); +} + +static DEFINE_MUTEX(device_links_lock); +DEFINE_STATIC_SRCU(device_links_srcu); + +static inline void device_links_write_lock(void) +{ + mutex_lock(&device_links_lock); +} + +static inline void device_links_write_unlock(void) +{ + mutex_unlock(&device_links_lock); +} + +int device_links_read_lock(void) __acquires(&device_links_srcu) +{ + return srcu_read_lock(&device_links_srcu); +} + +void device_links_read_unlock(int idx) __releases(&device_links_srcu) +{ + srcu_read_unlock(&device_links_srcu, idx); +} + +int device_links_read_lock_held(void) +{ + return srcu_read_lock_held(&device_links_srcu); +} + +static void device_link_synchronize_removal(void) +{ + synchronize_srcu(&device_links_srcu); +} + +static void device_link_remove_from_lists(struct device_link *link) +{ + list_del_rcu(&link->s_node); + list_del_rcu(&link->c_node); +} + +static bool device_is_ancestor(struct device *dev, struct device *target) +{ + while (target->parent) { + target = target->parent; + if (dev == target) + return true; + } + return false; +} + +#define DL_MARKER_FLAGS (DL_FLAG_INFERRED | \ + DL_FLAG_CYCLE | \ + DL_FLAG_MANAGED) +bool device_link_flag_is_sync_state_only(u32 flags) +{ + return (flags & ~DL_MARKER_FLAGS) == DL_FLAG_SYNC_STATE_ONLY; +} + +/** + * device_is_dependent - Check if one device depends on another one + * @dev: Device to check dependencies for. + * @target: Device to check against. + * + * Check if @target depends on @dev or any device dependent on it (its child or + * its consumer etc). Return 1 if that is the case or 0 otherwise. + */ +static int device_is_dependent(struct device *dev, void *target) +{ + struct device_link *link; + int ret; + + /* + * The "ancestors" check is needed to catch the case when the target + * device has not been completely initialized yet and it is still + * missing from the list of children of its parent device. + */ + if (dev == target || device_is_ancestor(dev, target)) + return 1; + + ret = device_for_each_child(dev, target, device_is_dependent); + if (ret) + return ret; + + list_for_each_entry(link, &dev->links.consumers, s_node) { + if (device_link_flag_is_sync_state_only(link->flags)) + continue; + + if (link->consumer == target) + return 1; + + ret = device_is_dependent(link->consumer, target); + if (ret) + break; + } + return ret; +} + +static void device_link_init_status(struct device_link *link, + struct device *consumer, + struct device *supplier) +{ + switch (supplier->links.status) { + case DL_DEV_PROBING: + switch (consumer->links.status) { + case DL_DEV_PROBING: + /* + * A consumer driver can create a link to a supplier + * that has not completed its probing yet as long as it + * knows that the supplier is already functional (for + * example, it has just acquired some resources from the + * supplier). + */ + link->status = DL_STATE_CONSUMER_PROBE; + break; + default: + link->status = DL_STATE_DORMANT; + break; + } + break; + case DL_DEV_DRIVER_BOUND: + switch (consumer->links.status) { + case DL_DEV_PROBING: + link->status = DL_STATE_CONSUMER_PROBE; + break; + case DL_DEV_DRIVER_BOUND: + link->status = DL_STATE_ACTIVE; + break; + default: + link->status = DL_STATE_AVAILABLE; + break; + } + break; + case DL_DEV_UNBINDING: + link->status = DL_STATE_SUPPLIER_UNBIND; + break; + default: + link->status = DL_STATE_DORMANT; + break; + } +} + +static int device_reorder_to_tail(struct device *dev, void *not_used) +{ + struct device_link *link; + + /* + * Devices that have not been registered yet will be put to the ends + * of the lists during the registration, so skip them here. + */ + if (device_is_registered(dev)) + devices_kset_move_last(dev); + + if (device_pm_initialized(dev)) + device_pm_move_last(dev); + + device_for_each_child(dev, NULL, device_reorder_to_tail); + list_for_each_entry(link, &dev->links.consumers, s_node) { + if (device_link_flag_is_sync_state_only(link->flags)) + continue; + device_reorder_to_tail(link->consumer, NULL); + } + + return 0; +} + +/** + * device_pm_move_to_tail - Move set of devices to the end of device lists + * @dev: Device to move + * + * This is a device_reorder_to_tail() wrapper taking the requisite locks. + * + * It moves the @dev along with all of its children and all of its consumers + * to the ends of the device_kset and dpm_list, recursively. + */ +void device_pm_move_to_tail(struct device *dev) +{ + int idx; + + idx = device_links_read_lock(); + device_pm_lock(); + device_reorder_to_tail(dev, NULL); + device_pm_unlock(); + device_links_read_unlock(idx); +} + +#define to_devlink(dev) container_of((dev), struct device_link, link_dev) + +static ssize_t status_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + const char *output; + + switch (to_devlink(dev)->status) { + case DL_STATE_NONE: + output = "not tracked"; + break; + case DL_STATE_DORMANT: + output = "dormant"; + break; + case DL_STATE_AVAILABLE: + output = "available"; + break; + case DL_STATE_CONSUMER_PROBE: + output = "consumer probing"; + break; + case DL_STATE_ACTIVE: + output = "active"; + break; + case DL_STATE_SUPPLIER_UNBIND: + output = "supplier unbinding"; + break; + default: + output = "unknown"; + break; + } + + return sysfs_emit(buf, "%s\n", output); +} +static DEVICE_ATTR_RO(status); + +static ssize_t auto_remove_on_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct device_link *link = to_devlink(dev); + const char *output; + + if (device_link_test(link, DL_FLAG_AUTOREMOVE_SUPPLIER)) + output = "supplier unbind"; + else if (device_link_test(link, DL_FLAG_AUTOREMOVE_CONSUMER)) + output = "consumer unbind"; + else + output = "never"; + + return sysfs_emit(buf, "%s\n", output); +} +static DEVICE_ATTR_RO(auto_remove_on); + +static ssize_t runtime_pm_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct device_link *link = to_devlink(dev); + + return sysfs_emit(buf, "%d\n", device_link_test(link, DL_FLAG_PM_RUNTIME)); +} +static DEVICE_ATTR_RO(runtime_pm); + +static ssize_t sync_state_only_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct device_link *link = to_devlink(dev); + + return sysfs_emit(buf, "%d\n", device_link_test(link, DL_FLAG_SYNC_STATE_ONLY)); +} +static DEVICE_ATTR_RO(sync_state_only); + +static struct attribute *devlink_attrs[] = { + &dev_attr_status.attr, + &dev_attr_auto_remove_on.attr, + &dev_attr_runtime_pm.attr, + &dev_attr_sync_state_only.attr, + NULL, +}; +ATTRIBUTE_GROUPS(devlink); + +static void device_link_release_fn(struct work_struct *work) +{ + struct device_link *link = container_of(work, struct device_link, rm_work); + + /* Ensure that all references to the link object have been dropped. */ + device_link_synchronize_removal(); + + pm_runtime_release_supplier(link); + /* + * If supplier_preactivated is set, the link has been dropped between + * the pm_runtime_get_suppliers() and pm_runtime_put_suppliers() calls + * in __driver_probe_device(). In that case, drop the supplier's + * PM-runtime usage counter to remove the reference taken by + * pm_runtime_get_suppliers(). + */ + if (link->supplier_preactivated) + pm_runtime_put_noidle(link->supplier); + + pm_request_idle(link->supplier); + + put_device(link->consumer); + put_device(link->supplier); + kfree(link); +} + +static void devlink_dev_release(struct device *dev) +{ + struct device_link *link = to_devlink(dev); + + INIT_WORK(&link->rm_work, device_link_release_fn); + /* + * It may take a while to complete this work because of the SRCU + * synchronization in device_link_release_fn() and if the consumer or + * supplier devices get deleted when it runs, so put it into the + * dedicated workqueue. + */ + queue_work(device_link_wq, &link->rm_work); +} + +/** + * device_link_wait_removal - Wait for ongoing devlink removal jobs to terminate + */ +void device_link_wait_removal(void) +{ + /* + * devlink removal jobs are queued in the dedicated work queue. + * To be sure that all removal jobs are terminated, ensure that any + * scheduled work has run to completion. + */ + flush_workqueue(device_link_wq); +} +EXPORT_SYMBOL_GPL(device_link_wait_removal); + +static const struct class devlink_class = { + .name = "devlink", + .dev_groups = devlink_groups, + .dev_release = devlink_dev_release, +}; + +static int devlink_add_symlinks(struct device *dev) +{ + char *buf_con __free(kfree) = NULL, *buf_sup __free(kfree) = NULL; + int ret; + struct device_link *link = to_devlink(dev); + struct device *sup = link->supplier; + struct device *con = link->consumer; + + ret = sysfs_create_link(&link->link_dev.kobj, &sup->kobj, "supplier"); + if (ret) + goto out; + + ret = sysfs_create_link(&link->link_dev.kobj, &con->kobj, "consumer"); + if (ret) + goto err_con; + + buf_con = kasprintf(GFP_KERNEL, "consumer:%s:%s", dev_bus_name(con), dev_name(con)); + if (!buf_con) { + ret = -ENOMEM; + goto err_con_dev; + } + + ret = sysfs_create_link(&sup->kobj, &link->link_dev.kobj, buf_con); + if (ret) + goto err_con_dev; + + buf_sup = kasprintf(GFP_KERNEL, "supplier:%s:%s", dev_bus_name(sup), dev_name(sup)); + if (!buf_sup) { + ret = -ENOMEM; + goto err_sup_dev; + } + + ret = sysfs_create_link(&con->kobj, &link->link_dev.kobj, buf_sup); + if (ret) + goto err_sup_dev; + + goto out; + +err_sup_dev: + sysfs_remove_link(&sup->kobj, buf_con); +err_con_dev: + sysfs_remove_link(&link->link_dev.kobj, "consumer"); +err_con: + sysfs_remove_link(&link->link_dev.kobj, "supplier"); +out: + return ret; +} + +static void devlink_remove_symlinks(struct device *dev) +{ + char *buf_con __free(kfree) = NULL, *buf_sup __free(kfree) = NULL; + struct device_link *link = to_devlink(dev); + struct device *sup = link->supplier; + struct device *con = link->consumer; + + sysfs_remove_link(&link->link_dev.kobj, "consumer"); + sysfs_remove_link(&link->link_dev.kobj, "supplier"); + + if (device_is_registered(con)) { + buf_sup = kasprintf(GFP_KERNEL, "supplier:%s:%s", dev_bus_name(sup), dev_name(sup)); + if (!buf_sup) + goto out; + sysfs_remove_link(&con->kobj, buf_sup); + } + + buf_con = kasprintf(GFP_KERNEL, "consumer:%s:%s", dev_bus_name(con), dev_name(con)); + if (!buf_con) + goto out; + sysfs_remove_link(&sup->kobj, buf_con); + + return; + +out: + WARN(1, "Unable to properly free device link symlinks!\n"); +} + +static struct class_interface devlink_class_intf = { + .class = &devlink_class, + .add_dev = devlink_add_symlinks, + .remove_dev = devlink_remove_symlinks, +}; + +static int __init devlink_class_init(void) +{ + int ret; + + ret = class_register(&devlink_class); + if (ret) + return ret; + + ret = class_interface_register(&devlink_class_intf); + if (ret) + class_unregister(&devlink_class); + + return ret; +} +postcore_initcall(devlink_class_init); + +#define DL_MANAGED_LINK_FLAGS (DL_FLAG_AUTOREMOVE_CONSUMER | \ + DL_FLAG_AUTOREMOVE_SUPPLIER | \ + DL_FLAG_AUTOPROBE_CONSUMER | \ + DL_FLAG_SYNC_STATE_ONLY | \ + DL_FLAG_INFERRED | \ + DL_FLAG_CYCLE) + +#define DL_ADD_VALID_FLAGS (DL_MANAGED_LINK_FLAGS | DL_FLAG_STATELESS | \ + DL_FLAG_PM_RUNTIME | DL_FLAG_RPM_ACTIVE) + +/** + * device_link_add - Create a link between two devices. + * @consumer: Consumer end of the link. + * @supplier: Supplier end of the link. + * @flags: Link flags. + * + * Return: On success, a device_link struct will be returned. + * On error or invalid flag settings, NULL will be returned. + * + * The caller is responsible for the proper synchronization of the link creation + * with runtime PM. First, setting the DL_FLAG_PM_RUNTIME flag will cause the + * runtime PM framework to take the link into account. Second, if the + * DL_FLAG_RPM_ACTIVE flag is set in addition to it, the supplier devices will + * be forced into the active meta state and reference-counted upon the creation + * of the link. If DL_FLAG_PM_RUNTIME is not set, DL_FLAG_RPM_ACTIVE will be + * ignored. + * + * If DL_FLAG_STATELESS is set in @flags, the caller of this function is + * expected to release the link returned by it directly with the help of either + * device_link_del() or device_link_remove(). + * + * If that flag is not set, however, the caller of this function is handing the + * management of the link over to the driver core entirely and its return value + * can only be used to check whether or not the link is present. In that case, + * the DL_FLAG_AUTOREMOVE_CONSUMER and DL_FLAG_AUTOREMOVE_SUPPLIER device link + * flags can be used to indicate to the driver core when the link can be safely + * deleted. Namely, setting one of them in @flags indicates to the driver core + * that the link is not going to be used (by the given caller of this function) + * after unbinding the consumer or supplier driver, respectively, from its + * device, so the link can be deleted at that point. If none of them is set, + * the link will be maintained until one of the devices pointed to by it (either + * the consumer or the supplier) is unregistered. + * + * Also, if DL_FLAG_STATELESS, DL_FLAG_AUTOREMOVE_CONSUMER and + * DL_FLAG_AUTOREMOVE_SUPPLIER are not set in @flags (that is, a persistent + * managed device link is being added), the DL_FLAG_AUTOPROBE_CONSUMER flag can + * be used to request the driver core to automatically probe for a consumer + * driver after successfully binding a driver to the supplier device. + * + * The combination of DL_FLAG_STATELESS and one of DL_FLAG_AUTOREMOVE_CONSUMER, + * DL_FLAG_AUTOREMOVE_SUPPLIER, or DL_FLAG_AUTOPROBE_CONSUMER set in @flags at + * the same time is invalid and will cause NULL to be returned upfront. + * However, if a device link between the given @consumer and @supplier pair + * exists already when this function is called for them, the existing link will + * be returned regardless of its current type and status (the link's flags may + * be modified then). The caller of this function is then expected to treat + * the link as though it has just been created, so (in particular) if + * DL_FLAG_STATELESS was passed in @flags, the link needs to be released + * explicitly when not needed any more (as stated above). + * + * A side effect of the link creation is re-ordering of dpm_list and the + * devices_kset list by moving the consumer device and all devices depending + * on it to the ends of these lists (that does not happen to devices that have + * not been registered when this function is called). + * + * The supplier device is required to be registered when this function is called + * and NULL will be returned if that is not the case. The consumer device need + * not be registered, however. + */ +struct device_link *device_link_add(struct device *consumer, + struct device *supplier, u32 flags) +{ + struct device_link *link; + + if (!consumer || !supplier || consumer == supplier || + flags & ~DL_ADD_VALID_FLAGS || + (flags & DL_FLAG_STATELESS && flags & DL_MANAGED_LINK_FLAGS) || + (flags & DL_FLAG_AUTOPROBE_CONSUMER && + flags & (DL_FLAG_AUTOREMOVE_CONSUMER | + DL_FLAG_AUTOREMOVE_SUPPLIER))) + return NULL; + + if (flags & DL_FLAG_PM_RUNTIME && flags & DL_FLAG_RPM_ACTIVE) { + if (pm_runtime_get_sync(supplier) < 0) { + pm_runtime_put_noidle(supplier); + return NULL; + } + } + + if (!(flags & DL_FLAG_STATELESS)) + flags |= DL_FLAG_MANAGED; + + if (flags & DL_FLAG_SYNC_STATE_ONLY && + !device_link_flag_is_sync_state_only(flags)) + return NULL; + + device_links_write_lock(); + device_pm_lock(); + + /* + * If the supplier has not been fully registered yet or there is a + * reverse (non-SYNC_STATE_ONLY) dependency between the consumer and + * the supplier already in the graph, return NULL. If the link is a + * SYNC_STATE_ONLY link, we don't check for reverse dependencies + * because it only affects sync_state() callbacks. + */ + if (!device_pm_initialized(supplier) + || (!(flags & DL_FLAG_SYNC_STATE_ONLY) && + device_is_dependent(consumer, supplier))) { + link = NULL; + goto out; + } + + /* + * SYNC_STATE_ONLY links are useless once a consumer device has probed. + * So, only create it if the consumer hasn't probed yet. + */ + if (flags & DL_FLAG_SYNC_STATE_ONLY && + consumer->links.status != DL_DEV_NO_DRIVER && + consumer->links.status != DL_DEV_PROBING) { + link = NULL; + goto out; + } + + /* + * DL_FLAG_AUTOREMOVE_SUPPLIER indicates that the link will be needed + * longer than for DL_FLAG_AUTOREMOVE_CONSUMER and setting them both + * together doesn't make sense, so prefer DL_FLAG_AUTOREMOVE_SUPPLIER. + */ + if (flags & DL_FLAG_AUTOREMOVE_SUPPLIER) + flags &= ~DL_FLAG_AUTOREMOVE_CONSUMER; + + list_for_each_entry(link, &supplier->links.consumers, s_node) { + if (link->consumer != consumer) + continue; + + if (device_link_test(link, DL_FLAG_INFERRED) && + !(flags & DL_FLAG_INFERRED)) + link->flags &= ~DL_FLAG_INFERRED; + + if (flags & DL_FLAG_PM_RUNTIME) { + if (!device_link_test(link, DL_FLAG_PM_RUNTIME)) { + pm_runtime_new_link(consumer); + link->flags |= DL_FLAG_PM_RUNTIME; + } + if (flags & DL_FLAG_RPM_ACTIVE) + refcount_inc(&link->rpm_active); + } + + if (flags & DL_FLAG_STATELESS) { + kref_get(&link->kref); + if (device_link_test(link, DL_FLAG_SYNC_STATE_ONLY) && + !device_link_test(link, DL_FLAG_STATELESS)) { + link->flags |= DL_FLAG_STATELESS; + goto reorder; + } else { + link->flags |= DL_FLAG_STATELESS; + goto out; + } + } + + /* + * If the life time of the link following from the new flags is + * longer than indicated by the flags of the existing link, + * update the existing link to stay around longer. + */ + if (flags & DL_FLAG_AUTOREMOVE_SUPPLIER) { + if (device_link_test(link, DL_FLAG_AUTOREMOVE_CONSUMER)) { + link->flags &= ~DL_FLAG_AUTOREMOVE_CONSUMER; + link->flags |= DL_FLAG_AUTOREMOVE_SUPPLIER; + } + } else if (!(flags & DL_FLAG_AUTOREMOVE_CONSUMER)) { + link->flags &= ~(DL_FLAG_AUTOREMOVE_CONSUMER | + DL_FLAG_AUTOREMOVE_SUPPLIER); + } + if (!device_link_test(link, DL_FLAG_MANAGED)) { + kref_get(&link->kref); + link->flags |= DL_FLAG_MANAGED; + device_link_init_status(link, consumer, supplier); + } + if (device_link_test(link, DL_FLAG_SYNC_STATE_ONLY) && + !(flags & DL_FLAG_SYNC_STATE_ONLY)) { + link->flags &= ~DL_FLAG_SYNC_STATE_ONLY; + goto reorder; + } + + goto out; + } + + link = kzalloc(sizeof(*link), GFP_KERNEL); + if (!link) + goto out; + + refcount_set(&link->rpm_active, 1); + + get_device(supplier); + link->supplier = supplier; + INIT_LIST_HEAD(&link->s_node); + get_device(consumer); + link->consumer = consumer; + INIT_LIST_HEAD(&link->c_node); + link->flags = flags; + kref_init(&link->kref); + + link->link_dev.class = &devlink_class; + device_set_pm_not_required(&link->link_dev); + dev_set_name(&link->link_dev, "%s:%s--%s:%s", + dev_bus_name(supplier), dev_name(supplier), + dev_bus_name(consumer), dev_name(consumer)); + if (device_register(&link->link_dev)) { + put_device(&link->link_dev); + link = NULL; + goto out; + } + + if (flags & DL_FLAG_PM_RUNTIME) { + if (flags & DL_FLAG_RPM_ACTIVE) + refcount_inc(&link->rpm_active); + + pm_runtime_new_link(consumer); + } + + /* Determine the initial link state. */ + if (flags & DL_FLAG_STATELESS) + link->status = DL_STATE_NONE; + else + device_link_init_status(link, consumer, supplier); + + /* + * Some callers expect the link creation during consumer driver probe to + * resume the supplier even without DL_FLAG_RPM_ACTIVE. + */ + if (link->status == DL_STATE_CONSUMER_PROBE && + flags & DL_FLAG_PM_RUNTIME) + pm_runtime_resume(supplier); + + list_add_tail_rcu(&link->s_node, &supplier->links.consumers); + list_add_tail_rcu(&link->c_node, &consumer->links.suppliers); + + if (flags & DL_FLAG_SYNC_STATE_ONLY) { + dev_dbg(consumer, + "Linked as a sync state only consumer to %s\n", + dev_name(supplier)); + goto out; + } + +reorder: + /* + * Move the consumer and all of the devices depending on it to the end + * of dpm_list and the devices_kset list. + * + * It is necessary to hold dpm_list locked throughout all that or else + * we may end up suspending with a wrong ordering of it. + */ + device_reorder_to_tail(consumer, NULL); + + dev_dbg(consumer, "Linked as a consumer to %s\n", dev_name(supplier)); + +out: + device_pm_unlock(); + device_links_write_unlock(); + + if ((flags & DL_FLAG_PM_RUNTIME && flags & DL_FLAG_RPM_ACTIVE) && !link) + pm_runtime_put(supplier); + + return link; +} +EXPORT_SYMBOL_GPL(device_link_add); + +static void __device_link_del(struct kref *kref) +{ + struct device_link *link = container_of(kref, struct device_link, kref); + + dev_dbg(link->consumer, "Dropping the link to %s\n", + dev_name(link->supplier)); + + pm_runtime_drop_link(link); + + device_link_remove_from_lists(link); + device_unregister(&link->link_dev); +} + +static void device_link_put_kref(struct device_link *link) +{ + if (device_link_test(link, DL_FLAG_STATELESS)) + kref_put(&link->kref, __device_link_del); + else if (!device_is_registered(link->consumer)) + __device_link_del(&link->kref); + else + WARN(1, "Unable to drop a managed device link reference\n"); +} + +/** + * device_link_del - Delete a stateless link between two devices. + * @link: Device link to delete. + * + * The caller must ensure proper synchronization of this function with runtime + * PM. If the link was added multiple times, it needs to be deleted as often. + * Care is required for hotplugged devices: Their links are purged on removal + * and calling device_link_del() is then no longer allowed. + */ +void device_link_del(struct device_link *link) +{ + device_links_write_lock(); + device_link_put_kref(link); + device_links_write_unlock(); +} +EXPORT_SYMBOL_GPL(device_link_del); + +/** + * device_link_remove - Delete a stateless link between two devices. + * @consumer: Consumer end of the link. + * @supplier: Supplier end of the link. + * + * The caller must ensure proper synchronization of this function with runtime + * PM. + */ +void device_link_remove(void *consumer, struct device *supplier) +{ + struct device_link *link; + + if (WARN_ON(consumer == supplier)) + return; + + device_links_write_lock(); + + list_for_each_entry(link, &supplier->links.consumers, s_node) { + if (link->consumer == consumer) { + device_link_put_kref(link); + break; + } + } + + device_links_write_unlock(); +} +EXPORT_SYMBOL_GPL(device_link_remove); + +static void device_links_missing_supplier(struct device *dev) +{ + struct device_link *link; + + list_for_each_entry(link, &dev->links.suppliers, c_node) { + if (link->status != DL_STATE_CONSUMER_PROBE) + continue; + + if (link->supplier->links.status == DL_DEV_DRIVER_BOUND) { + WRITE_ONCE(link->status, DL_STATE_AVAILABLE); + } else { + WARN_ON(!device_link_test(link, DL_FLAG_SYNC_STATE_ONLY)); + WRITE_ONCE(link->status, DL_STATE_DORMANT); + } + } +} + +static bool dev_is_best_effort(struct device *dev) +{ + return (fw_devlink_best_effort && dev->can_match) || + (dev->fwnode && (dev->fwnode->flags & FWNODE_FLAG_BEST_EFFORT)); +} + +static struct fwnode_handle *fwnode_links_check_suppliers( + struct fwnode_handle *fwnode) +{ + struct fwnode_link *link; + + if (!fwnode || fw_devlink_is_permissive()) + return NULL; + + list_for_each_entry(link, &fwnode->suppliers, c_hook) + if (!(link->flags & + (FWLINK_FLAG_CYCLE | FWLINK_FLAG_IGNORE))) + return link->supplier; + + return NULL; +} + +/** + * device_links_check_suppliers - Check presence of supplier drivers. + * @dev: Consumer device. + * + * Check links from this device to any suppliers. Walk the list of the device's + * links to suppliers and see if all of them are available. If not, simply + * return -EPROBE_DEFER. + * + * We need to guarantee that the supplier will not go away after the check has + * been positive here. It only can go away in __device_release_driver() and + * that function checks the device's links to consumers. This means we need to + * mark the link as "consumer probe in progress" to make the supplier removal + * wait for us to complete (or bad things may happen). + * + * Links without the DL_FLAG_MANAGED flag set are ignored. + */ +int device_links_check_suppliers(struct device *dev) +{ + struct device_link *link; + int ret = 0, fwnode_ret = 0; + struct fwnode_handle *sup_fw; + + /* + * Device waiting for supplier to become available is not allowed to + * probe. + */ + scoped_guard(mutex, &fwnode_link_lock) { + sup_fw = fwnode_links_check_suppliers(dev->fwnode); + if (sup_fw) { + if (dev_is_best_effort(dev)) + fwnode_ret = -EAGAIN; + else + return dev_err_probe(dev, -EPROBE_DEFER, + "wait for supplier %pfwf\n", sup_fw); + } + } + + device_links_write_lock(); + + list_for_each_entry(link, &dev->links.suppliers, c_node) { + if (!device_link_test(link, DL_FLAG_MANAGED)) + continue; + + if (link->status != DL_STATE_AVAILABLE && + !device_link_test(link, DL_FLAG_SYNC_STATE_ONLY)) { + + if (dev_is_best_effort(dev) && + device_link_test(link, DL_FLAG_INFERRED) && + !link->supplier->can_match) { + ret = -EAGAIN; + continue; + } + + device_links_missing_supplier(dev); + ret = dev_err_probe(dev, -EPROBE_DEFER, + "supplier %s not ready\n", dev_name(link->supplier)); + break; + } + WRITE_ONCE(link->status, DL_STATE_CONSUMER_PROBE); + } + dev->links.status = DL_DEV_PROBING; + + device_links_write_unlock(); + + return ret ? ret : fwnode_ret; +} + +/** + * __device_links_queue_sync_state - Queue a device for sync_state() callback + * @dev: Device to call sync_state() on + * @list: List head to queue the @dev on + * + * Queues a device for a sync_state() callback when the device links write lock + * isn't held. This allows the sync_state() execution flow to use device links + * APIs. The caller must ensure this function is called with + * device_links_write_lock() held. + * + * This function does a get_device() to make sure the device is not freed while + * on this list. + * + * So the caller must also ensure that device_links_flush_sync_list() is called + * as soon as the caller releases device_links_write_lock(). This is necessary + * to make sure the sync_state() is called in a timely fashion and the + * put_device() is called on this device. + */ +static void __device_links_queue_sync_state(struct device *dev, + struct list_head *list) +{ + struct device_link *link; + + if (!dev_has_sync_state(dev)) + return; + if (dev->state_synced) + return; + + list_for_each_entry(link, &dev->links.consumers, s_node) { + if (!device_link_test(link, DL_FLAG_MANAGED)) + continue; + if (link->status != DL_STATE_ACTIVE) + return; + } + + /* + * Set the flag here to avoid adding the same device to a list more + * than once. This can happen if new consumers get added to the device + * and probed before the list is flushed. + */ + dev->state_synced = true; + + if (WARN_ON(!list_empty(&dev->links.defer_sync))) + return; + + get_device(dev); + list_add_tail(&dev->links.defer_sync, list); +} + +/** + * device_links_flush_sync_list - Call sync_state() on a list of devices + * @list: List of devices to call sync_state() on + * @dont_lock_dev: Device for which lock is already held by the caller + * + * Calls sync_state() on all the devices that have been queued for it. This + * function is used in conjunction with __device_links_queue_sync_state(). The + * @dont_lock_dev parameter is useful when this function is called from a + * context where a device lock is already held. + */ +static void device_links_flush_sync_list(struct list_head *list, + struct device *dont_lock_dev) +{ + struct device *dev, *tmp; + + list_for_each_entry_safe(dev, tmp, list, links.defer_sync) { + list_del_init(&dev->links.defer_sync); + + if (dev != dont_lock_dev) + device_lock(dev); + + dev_sync_state(dev); + + if (dev != dont_lock_dev) + device_unlock(dev); + + put_device(dev); + } +} + +void device_links_supplier_sync_state_pause(void) +{ + device_links_write_lock(); + defer_sync_state_count++; + device_links_write_unlock(); +} + +void device_links_supplier_sync_state_resume(void) +{ + struct device *dev, *tmp; + LIST_HEAD(sync_list); + + device_links_write_lock(); + if (!defer_sync_state_count) { + WARN(true, "Unmatched sync_state pause/resume!"); + goto out; + } + defer_sync_state_count--; + if (defer_sync_state_count) + goto out; + + list_for_each_entry_safe(dev, tmp, &deferred_sync, links.defer_sync) { + /* + * Delete from deferred_sync list before queuing it to + * sync_list because defer_sync is used for both lists. + */ + list_del_init(&dev->links.defer_sync); + __device_links_queue_sync_state(dev, &sync_list); + } +out: + device_links_write_unlock(); + + device_links_flush_sync_list(&sync_list, NULL); +} + +static int sync_state_resume_initcall(void) +{ + device_links_supplier_sync_state_resume(); + return 0; +} +late_initcall(sync_state_resume_initcall); + +static void __device_links_supplier_defer_sync(struct device *sup) +{ + if (list_empty(&sup->links.defer_sync) && dev_has_sync_state(sup)) + list_add_tail(&sup->links.defer_sync, &deferred_sync); +} + +static void device_link_drop_managed(struct device_link *link) +{ + link->flags &= ~DL_FLAG_MANAGED; + WRITE_ONCE(link->status, DL_STATE_NONE); + kref_put(&link->kref, __device_link_del); +} + +static ssize_t waiting_for_supplier_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + bool val; + + device_lock(dev); + scoped_guard(mutex, &fwnode_link_lock) + val = !!fwnode_links_check_suppliers(dev->fwnode); + device_unlock(dev); + return sysfs_emit(buf, "%u\n", val); +} +static DEVICE_ATTR_RO(waiting_for_supplier); + +/** + * device_links_force_bind - Prepares device to be force bound + * @dev: Consumer device. + * + * device_bind_driver() force binds a device to a driver without calling any + * driver probe functions. So the consumer really isn't going to wait for any + * supplier before it's bound to the driver. We still want the device link + * states to be sensible when this happens. + * + * In preparation for device_bind_driver(), this function goes through each + * supplier device links and checks if the supplier is bound. If it is, then + * the device link status is set to CONSUMER_PROBE. Otherwise, the device link + * is dropped. Links without the DL_FLAG_MANAGED flag set are ignored. + */ +void device_links_force_bind(struct device *dev) +{ + struct device_link *link, *ln; + + device_links_write_lock(); + + list_for_each_entry_safe(link, ln, &dev->links.suppliers, c_node) { + if (!device_link_test(link, DL_FLAG_MANAGED)) + continue; + + if (link->status != DL_STATE_AVAILABLE) { + device_link_drop_managed(link); + continue; + } + WRITE_ONCE(link->status, DL_STATE_CONSUMER_PROBE); + } + dev->links.status = DL_DEV_PROBING; + + device_links_write_unlock(); +} + +/** + * device_links_driver_bound - Update device links after probing its driver. + * @dev: Device to update the links for. + * + * The probe has been successful, so update links from this device to any + * consumers by changing their status to "available". + * + * Also change the status of @dev's links to suppliers to "active". + * + * Links without the DL_FLAG_MANAGED flag set are ignored. + */ +void device_links_driver_bound(struct device *dev) +{ + struct device_link *link, *ln; + LIST_HEAD(sync_list); + + /* + * If a device binds successfully, it's expected to have created all + * the device links it needs to or make new device links as it needs + * them. So, fw_devlink no longer needs to create device links to any + * of the device's suppliers. + * + * Also, if a child firmware node of this bound device is not added as a + * device by now, assume it is never going to be added. Make this bound + * device the fallback supplier to the dangling consumers of the child + * firmware node because this bound device is probably implementing the + * child firmware node functionality and we don't want the dangling + * consumers to defer probe indefinitely waiting for a device for the + * child firmware node. + */ + if (dev->fwnode && dev->fwnode->dev == dev) { + struct fwnode_handle *child; + + fwnode_links_purge_suppliers(dev->fwnode); + + guard(mutex)(&fwnode_link_lock); + + fwnode_for_each_available_child_node(dev->fwnode, child) + __fw_devlink_pickup_dangling_consumers(child, + dev->fwnode); + __fw_devlink_link_to_consumers(dev); + } + device_remove_file(dev, &dev_attr_waiting_for_supplier); + + device_links_write_lock(); + + list_for_each_entry(link, &dev->links.consumers, s_node) { + if (!device_link_test(link, DL_FLAG_MANAGED)) + continue; + + /* + * Links created during consumer probe may be in the "consumer + * probe" state to start with if the supplier is still probing + * when they are created and they may become "active" if the + * consumer probe returns first. Skip them here. + */ + if (link->status == DL_STATE_CONSUMER_PROBE || + link->status == DL_STATE_ACTIVE) + continue; + + WARN_ON(link->status != DL_STATE_DORMANT); + WRITE_ONCE(link->status, DL_STATE_AVAILABLE); + + if (device_link_test(link, DL_FLAG_AUTOPROBE_CONSUMER)) + driver_deferred_probe_add(link->consumer); + } + + if (defer_sync_state_count) + __device_links_supplier_defer_sync(dev); + else + __device_links_queue_sync_state(dev, &sync_list); + + list_for_each_entry_safe(link, ln, &dev->links.suppliers, c_node) { + struct device *supplier; + + if (!device_link_test(link, DL_FLAG_MANAGED)) + continue; + + supplier = link->supplier; + if (device_link_test(link, DL_FLAG_SYNC_STATE_ONLY)) { + /* + * When DL_FLAG_SYNC_STATE_ONLY is set, it means no + * other DL_MANAGED_LINK_FLAGS have been set. So, it's + * save to drop the managed link completely. + */ + device_link_drop_managed(link); + } else if (dev_is_best_effort(dev) && + device_link_test(link, DL_FLAG_INFERRED) && + link->status != DL_STATE_CONSUMER_PROBE && + !link->supplier->can_match) { + /* + * When dev_is_best_effort() is true, we ignore device + * links to suppliers that don't have a driver. If the + * consumer device still managed to probe, there's no + * point in maintaining a device link in a weird state + * (consumer probed before supplier). So delete it. + */ + device_link_drop_managed(link); + } else { + WARN_ON(link->status != DL_STATE_CONSUMER_PROBE); + WRITE_ONCE(link->status, DL_STATE_ACTIVE); + } + + /* + * This needs to be done even for the deleted + * DL_FLAG_SYNC_STATE_ONLY device link in case it was the last + * device link that was preventing the supplier from getting a + * sync_state() call. + */ + if (defer_sync_state_count) + __device_links_supplier_defer_sync(supplier); + else + __device_links_queue_sync_state(supplier, &sync_list); + } + + dev->links.status = DL_DEV_DRIVER_BOUND; + + device_links_write_unlock(); + + device_links_flush_sync_list(&sync_list, dev); +} + +/** + * __device_links_no_driver - Update links of a device without a driver. + * @dev: Device without a drvier. + * + * Delete all non-persistent links from this device to any suppliers. + * + * Persistent links stay around, but their status is changed to "available", + * unless they already are in the "supplier unbind in progress" state in which + * case they need not be updated. + * + * Links without the DL_FLAG_MANAGED flag set are ignored. + */ +static void __device_links_no_driver(struct device *dev) +{ + struct device_link *link, *ln; + + list_for_each_entry_safe_reverse(link, ln, &dev->links.suppliers, c_node) { + if (!device_link_test(link, DL_FLAG_MANAGED)) + continue; + + if (device_link_test(link, DL_FLAG_AUTOREMOVE_CONSUMER)) { + device_link_drop_managed(link); + continue; + } + + if (link->status != DL_STATE_CONSUMER_PROBE && + link->status != DL_STATE_ACTIVE) + continue; + + if (link->supplier->links.status == DL_DEV_DRIVER_BOUND) { + WRITE_ONCE(link->status, DL_STATE_AVAILABLE); + } else { + WARN_ON(!device_link_test(link, DL_FLAG_SYNC_STATE_ONLY)); + WRITE_ONCE(link->status, DL_STATE_DORMANT); + } + } + + dev->links.status = DL_DEV_NO_DRIVER; +} + +/** + * device_links_no_driver - Update links after failing driver probe. + * @dev: Device whose driver has just failed to probe. + * + * Clean up leftover links to consumers for @dev and invoke + * %__device_links_no_driver() to update links to suppliers for it as + * appropriate. + * + * Links without the DL_FLAG_MANAGED flag set are ignored. + */ +void device_links_no_driver(struct device *dev) +{ + struct device_link *link; + + device_links_write_lock(); + + list_for_each_entry(link, &dev->links.consumers, s_node) { + if (!device_link_test(link, DL_FLAG_MANAGED)) + continue; + + /* + * The probe has failed, so if the status of the link is + * "consumer probe" or "active", it must have been added by + * a probing consumer while this device was still probing. + * Change its state to "dormant", as it represents a valid + * relationship, but it is not functionally meaningful. + */ + if (link->status == DL_STATE_CONSUMER_PROBE || + link->status == DL_STATE_ACTIVE) + WRITE_ONCE(link->status, DL_STATE_DORMANT); + } + + __device_links_no_driver(dev); + + device_links_write_unlock(); +} + +/** + * device_links_driver_cleanup - Update links after driver removal. + * @dev: Device whose driver has just gone away. + * + * Update links to consumers for @dev by changing their status to "dormant" and + * invoke %__device_links_no_driver() to update links to suppliers for it as + * appropriate. + * + * Links without the DL_FLAG_MANAGED flag set are ignored. + */ +void device_links_driver_cleanup(struct device *dev) +{ + struct device_link *link, *ln; + + device_links_write_lock(); + + list_for_each_entry_safe(link, ln, &dev->links.consumers, s_node) { + if (!device_link_test(link, DL_FLAG_MANAGED)) + continue; + + WARN_ON(device_link_test(link, DL_FLAG_AUTOREMOVE_CONSUMER)); + WARN_ON(link->status != DL_STATE_SUPPLIER_UNBIND); + + /* + * autoremove the links between this @dev and its consumer + * devices that are not active, i.e. where the link state + * has moved to DL_STATE_SUPPLIER_UNBIND. + */ + if (link->status == DL_STATE_SUPPLIER_UNBIND && + device_link_test(link, DL_FLAG_AUTOREMOVE_SUPPLIER)) + device_link_drop_managed(link); + + WRITE_ONCE(link->status, DL_STATE_DORMANT); + } + + list_del_init(&dev->links.defer_sync); + __device_links_no_driver(dev); + + device_links_write_unlock(); +} + +/** + * device_links_busy - Check if there are any busy links to consumers. + * @dev: Device to check. + * + * Check each consumer of the device and return 'true' if its link's status + * is one of "consumer probe" or "active" (meaning that the given consumer is + * probing right now or its driver is present). Otherwise, change the link + * state to "supplier unbind" to prevent the consumer from being probed + * successfully going forward. + * + * Return 'false' if there are no probing or active consumers. + * + * Links without the DL_FLAG_MANAGED flag set are ignored. + */ +bool device_links_busy(struct device *dev) +{ + struct device_link *link; + bool ret = false; + + device_links_write_lock(); + + list_for_each_entry(link, &dev->links.consumers, s_node) { + if (!device_link_test(link, DL_FLAG_MANAGED)) + continue; + + if (link->status == DL_STATE_CONSUMER_PROBE + || link->status == DL_STATE_ACTIVE) { + ret = true; + break; + } + WRITE_ONCE(link->status, DL_STATE_SUPPLIER_UNBIND); + } + + dev->links.status = DL_DEV_UNBINDING; + + device_links_write_unlock(); + return ret; +} + +/** + * device_links_unbind_consumers - Force unbind consumers of the given device. + * @dev: Device to unbind the consumers of. + * + * Walk the list of links to consumers for @dev and if any of them is in the + * "consumer probe" state, wait for all device probes in progress to complete + * and start over. + * + * If that's not the case, change the status of the link to "supplier unbind" + * and check if the link was in the "active" state. If so, force the consumer + * driver to unbind and start over (the consumer will not re-probe as we have + * changed the state of the link already). + * + * Links without the DL_FLAG_MANAGED flag set are ignored. + */ +void device_links_unbind_consumers(struct device *dev) +{ + struct device_link *link; + + start: + device_links_write_lock(); + + list_for_each_entry(link, &dev->links.consumers, s_node) { + enum device_link_state status; + + if (!device_link_test(link, DL_FLAG_MANAGED) || + device_link_test(link, DL_FLAG_SYNC_STATE_ONLY)) + continue; + + status = link->status; + if (status == DL_STATE_CONSUMER_PROBE) { + device_links_write_unlock(); + + wait_for_device_probe(); + goto start; + } + WRITE_ONCE(link->status, DL_STATE_SUPPLIER_UNBIND); + if (status == DL_STATE_ACTIVE) { + struct device *consumer = link->consumer; + + get_device(consumer); + + device_links_write_unlock(); + + device_release_driver_internal(consumer, NULL, + consumer->parent); + put_device(consumer); + goto start; + } + } + + device_links_write_unlock(); +} + +/** + * device_links_purge - Delete existing links to other devices. + * @dev: Target device. + */ +static void device_links_purge(struct device *dev) +{ + struct device_link *link, *ln; + + if (dev->class == &devlink_class) + return; + + /* + * Delete all of the remaining links from this device to any other + * devices (either consumers or suppliers). + */ + device_links_write_lock(); + + list_for_each_entry_safe_reverse(link, ln, &dev->links.suppliers, c_node) { + WARN_ON(link->status == DL_STATE_ACTIVE); + __device_link_del(&link->kref); + } + + list_for_each_entry_safe_reverse(link, ln, &dev->links.consumers, s_node) { + WARN_ON(link->status != DL_STATE_DORMANT && + link->status != DL_STATE_NONE); + __device_link_del(&link->kref); + } + + device_links_write_unlock(); +} + +#define FW_DEVLINK_FLAGS_PERMISSIVE (DL_FLAG_INFERRED | \ + DL_FLAG_SYNC_STATE_ONLY) +#define FW_DEVLINK_FLAGS_ON (DL_FLAG_INFERRED | \ + DL_FLAG_AUTOPROBE_CONSUMER) +#define FW_DEVLINK_FLAGS_RPM (FW_DEVLINK_FLAGS_ON | \ + DL_FLAG_PM_RUNTIME) + +static u32 fw_devlink_flags = FW_DEVLINK_FLAGS_RPM; +static int __init fw_devlink_setup(char *arg) +{ + if (!arg) + return -EINVAL; + + if (strcmp(arg, "off") == 0) { + fw_devlink_flags = 0; + } else if (strcmp(arg, "permissive") == 0) { + fw_devlink_flags = FW_DEVLINK_FLAGS_PERMISSIVE; + } else if (strcmp(arg, "on") == 0) { + fw_devlink_flags = FW_DEVLINK_FLAGS_ON; + } else if (strcmp(arg, "rpm") == 0) { + fw_devlink_flags = FW_DEVLINK_FLAGS_RPM; + } + return 0; +} +early_param("fw_devlink", fw_devlink_setup); + +static bool fw_devlink_strict; +static int __init fw_devlink_strict_setup(char *arg) +{ + return kstrtobool(arg, &fw_devlink_strict); +} +early_param("fw_devlink.strict", fw_devlink_strict_setup); + +#define FW_DEVLINK_SYNC_STATE_STRICT 0 +#define FW_DEVLINK_SYNC_STATE_TIMEOUT 1 + +#ifndef CONFIG_FW_DEVLINK_SYNC_STATE_TIMEOUT +static int fw_devlink_sync_state; #else -long sysfs_deprecated = 0; +static int fw_devlink_sync_state = FW_DEVLINK_SYNC_STATE_TIMEOUT; #endif -static __init int sysfs_deprecated_setup(char *arg) + +static int __init fw_devlink_sync_state_setup(char *arg) { - return strict_strtol(arg, 10, &sysfs_deprecated); + if (!arg) + return -EINVAL; + + if (strcmp(arg, "strict") == 0) { + fw_devlink_sync_state = FW_DEVLINK_SYNC_STATE_STRICT; + return 0; + } else if (strcmp(arg, "timeout") == 0) { + fw_devlink_sync_state = FW_DEVLINK_SYNC_STATE_TIMEOUT; + return 0; + } + return -EINVAL; } -early_param("sysfs.deprecated", sysfs_deprecated_setup); -#endif +early_param("fw_devlink.sync_state", fw_devlink_sync_state_setup); + +static inline u32 fw_devlink_get_flags(u8 fwlink_flags) +{ + if (fwlink_flags & FWLINK_FLAG_CYCLE) + return FW_DEVLINK_FLAGS_PERMISSIVE | DL_FLAG_CYCLE; + + return fw_devlink_flags; +} + +static bool fw_devlink_is_permissive(void) +{ + return fw_devlink_flags == FW_DEVLINK_FLAGS_PERMISSIVE; +} + +bool fw_devlink_is_strict(void) +{ + return fw_devlink_strict && !fw_devlink_is_permissive(); +} + +static void fw_devlink_parse_fwnode(struct fwnode_handle *fwnode) +{ + if (fwnode->flags & FWNODE_FLAG_LINKS_ADDED) + return; + + fwnode_call_int_op(fwnode, add_links); + fwnode->flags |= FWNODE_FLAG_LINKS_ADDED; +} + +static void fw_devlink_parse_fwtree(struct fwnode_handle *fwnode) +{ + struct fwnode_handle *child = NULL; + + fw_devlink_parse_fwnode(fwnode); + + while ((child = fwnode_get_next_available_child_node(fwnode, child))) + fw_devlink_parse_fwtree(child); +} + +static void fw_devlink_relax_link(struct device_link *link) +{ + if (!device_link_test(link, DL_FLAG_INFERRED)) + return; + + if (device_link_flag_is_sync_state_only(link->flags)) + return; + + pm_runtime_drop_link(link); + link->flags = DL_FLAG_MANAGED | FW_DEVLINK_FLAGS_PERMISSIVE; + dev_dbg(link->consumer, "Relaxing link with %s\n", + dev_name(link->supplier)); +} + +static int fw_devlink_no_driver(struct device *dev, void *data) +{ + struct device_link *link = to_devlink(dev); + + if (!link->supplier->can_match) + fw_devlink_relax_link(link); + + return 0; +} + +void fw_devlink_drivers_done(void) +{ + fw_devlink_drv_reg_done = true; + device_links_write_lock(); + class_for_each_device(&devlink_class, NULL, NULL, + fw_devlink_no_driver); + device_links_write_unlock(); +} + +static int fw_devlink_dev_sync_state(struct device *dev, void *data) +{ + struct device_link *link = to_devlink(dev); + struct device *sup = link->supplier; + + if (!device_link_test(link, DL_FLAG_MANAGED) || + link->status == DL_STATE_ACTIVE || sup->state_synced || + !dev_has_sync_state(sup)) + return 0; + + if (fw_devlink_sync_state == FW_DEVLINK_SYNC_STATE_STRICT) { + dev_info(sup, "sync_state() pending due to %s\n", + dev_name(link->consumer)); + return 0; + } + + if (!list_empty(&sup->links.defer_sync)) + return 0; + + dev_warn(sup, "Timed out. Forcing sync_state()\n"); + sup->state_synced = true; + get_device(sup); + list_add_tail(&sup->links.defer_sync, data); + + return 0; +} + +void fw_devlink_probing_done(void) +{ + LIST_HEAD(sync_list); + + device_links_write_lock(); + class_for_each_device(&devlink_class, NULL, &sync_list, + fw_devlink_dev_sync_state); + device_links_write_unlock(); + device_links_flush_sync_list(&sync_list, NULL); +} + +/** + * wait_for_init_devices_probe - Try to probe any device needed for init + * + * Some devices might need to be probed and bound successfully before the kernel + * boot sequence can finish and move on to init/userspace. For example, a + * network interface might need to be bound to be able to mount a NFS rootfs. + * + * With fw_devlink=on by default, some of these devices might be blocked from + * probing because they are waiting on a optional supplier that doesn't have a + * driver. While fw_devlink will eventually identify such devices and unblock + * the probing automatically, it might be too late by the time it unblocks the + * probing of devices. For example, the IP4 autoconfig might timeout before + * fw_devlink unblocks probing of the network interface. + * + * This function is available to temporarily try and probe all devices that have + * a driver even if some of their suppliers haven't been added or don't have + * drivers. + * + * The drivers can then decide which of the suppliers are optional vs mandatory + * and probe the device if possible. By the time this function returns, all such + * "best effort" probes are guaranteed to be completed. If a device successfully + * probes in this mode, we delete all fw_devlink discovered dependencies of that + * device where the supplier hasn't yet probed successfully because they have to + * be optional dependencies. + * + * Any devices that didn't successfully probe go back to being treated as if + * this function was never called. + * + * This also means that some devices that aren't needed for init and could have + * waited for their optional supplier to probe (when the supplier's module is + * loaded later on) would end up probing prematurely with limited functionality. + * So call this function only when boot would fail without it. + */ +void __init wait_for_init_devices_probe(void) +{ + if (!fw_devlink_flags || fw_devlink_is_permissive()) + return; + + /* + * Wait for all ongoing probes to finish so that the "best effort" is + * only applied to devices that can't probe otherwise. + */ + wait_for_device_probe(); + + pr_info("Trying to probe devices needed for running init ...\n"); + fw_devlink_best_effort = true; + driver_deferred_probe_trigger(); + + /* + * Wait for all "best effort" probes to finish before going back to + * normal enforcement. + */ + wait_for_device_probe(); + fw_devlink_best_effort = false; +} + +static void fw_devlink_unblock_consumers(struct device *dev) +{ + struct device_link *link; + + if (!fw_devlink_flags || fw_devlink_is_permissive()) + return; + + device_links_write_lock(); + list_for_each_entry(link, &dev->links.consumers, s_node) + fw_devlink_relax_link(link); + device_links_write_unlock(); +} + +static bool fwnode_init_without_drv(struct fwnode_handle *fwnode) +{ + struct device *dev; + bool ret; + + if (!(fwnode->flags & FWNODE_FLAG_INITIALIZED)) + return false; + + dev = get_dev_from_fwnode(fwnode); + ret = !dev || dev->links.status == DL_DEV_NO_DRIVER; + put_device(dev); + + return ret; +} + +static bool fwnode_ancestor_init_without_drv(struct fwnode_handle *fwnode) +{ + struct fwnode_handle *parent; + + fwnode_for_each_parent_node(fwnode, parent) { + if (fwnode_init_without_drv(parent)) { + fwnode_handle_put(parent); + return true; + } + } + + return false; +} + +/** + * fwnode_is_ancestor_of - Test if @ancestor is ancestor of @child + * @ancestor: Firmware which is tested for being an ancestor + * @child: Firmware which is tested for being the child + * + * A node is considered an ancestor of itself too. + * + * Return: true if @ancestor is an ancestor of @child. Otherwise, returns false. + */ +static bool fwnode_is_ancestor_of(const struct fwnode_handle *ancestor, + const struct fwnode_handle *child) +{ + struct fwnode_handle *parent; + + if (IS_ERR_OR_NULL(ancestor)) + return false; + + if (child == ancestor) + return true; + + fwnode_for_each_parent_node(child, parent) { + if (parent == ancestor) { + fwnode_handle_put(parent); + return true; + } + } + return false; +} + +/** + * fwnode_get_next_parent_dev - Find device of closest ancestor fwnode + * @fwnode: firmware node + * + * Given a firmware node (@fwnode), this function finds its closest ancestor + * firmware node that has a corresponding struct device and returns that struct + * device. + * + * The caller is responsible for calling put_device() on the returned device + * pointer. + * + * Return: a pointer to the device of the @fwnode's closest ancestor. + */ +static struct device *fwnode_get_next_parent_dev(const struct fwnode_handle *fwnode) +{ + struct fwnode_handle *parent; + struct device *dev; + + fwnode_for_each_parent_node(fwnode, parent) { + dev = get_dev_from_fwnode(parent); + if (dev) { + fwnode_handle_put(parent); + return dev; + } + } + return NULL; +} + +/** + * __fw_devlink_relax_cycles - Relax and mark dependency cycles. + * @con_handle: Potential consumer device fwnode. + * @sup_handle: Potential supplier's fwnode. + * + * Needs to be called with fwnode_lock and device link lock held. + * + * Check if @sup_handle or any of its ancestors or suppliers direct/indirectly + * depend on @con. This function can detect multiple cyles between @sup_handle + * and @con. When such dependency cycles are found, convert all device links + * created solely by fw_devlink into SYNC_STATE_ONLY device links. Also, mark + * all fwnode links in the cycle with FWLINK_FLAG_CYCLE so that when they are + * converted into a device link in the future, they are created as + * SYNC_STATE_ONLY device links. This is the equivalent of doing + * fw_devlink=permissive just between the devices in the cycle. We need to do + * this because, at this point, fw_devlink can't tell which of these + * dependencies is not a real dependency. + * + * Return true if one or more cycles were found. Otherwise, return false. + */ +static bool __fw_devlink_relax_cycles(struct fwnode_handle *con_handle, + struct fwnode_handle *sup_handle) +{ + struct device *sup_dev = NULL, *par_dev = NULL, *con_dev = NULL; + struct fwnode_link *link; + struct device_link *dev_link; + bool ret = false; + + if (!sup_handle) + return false; + + /* + * We aren't trying to find all cycles. Just a cycle between con and + * sup_handle. + */ + if (sup_handle->flags & FWNODE_FLAG_VISITED) + return false; + + sup_handle->flags |= FWNODE_FLAG_VISITED; + + /* Termination condition. */ + if (sup_handle == con_handle) { + pr_debug("----- cycle: start -----\n"); + ret = true; + goto out; + } + + sup_dev = get_dev_from_fwnode(sup_handle); + con_dev = get_dev_from_fwnode(con_handle); + /* + * If sup_dev is bound to a driver and @con hasn't started binding to a + * driver, sup_dev can't be a consumer of @con. So, no need to check + * further. + */ + if (sup_dev && sup_dev->links.status == DL_DEV_DRIVER_BOUND && + con_dev && con_dev->links.status == DL_DEV_NO_DRIVER) { + ret = false; + goto out; + } + + list_for_each_entry(link, &sup_handle->suppliers, c_hook) { + if (link->flags & FWLINK_FLAG_IGNORE) + continue; + + if (__fw_devlink_relax_cycles(con_handle, link->supplier)) { + __fwnode_link_cycle(link); + ret = true; + } + } + + /* + * Give priority to device parent over fwnode parent to account for any + * quirks in how fwnodes are converted to devices. + */ + if (sup_dev) + par_dev = get_device(sup_dev->parent); + else + par_dev = fwnode_get_next_parent_dev(sup_handle); + + if (par_dev && __fw_devlink_relax_cycles(con_handle, par_dev->fwnode)) { + pr_debug("%pfwf: cycle: child of %pfwf\n", sup_handle, + par_dev->fwnode); + ret = true; + } + + if (!sup_dev) + goto out; + + list_for_each_entry(dev_link, &sup_dev->links.suppliers, c_node) { + /* + * Ignore a SYNC_STATE_ONLY flag only if it wasn't marked as + * such due to a cycle. + */ + if (device_link_flag_is_sync_state_only(dev_link->flags) && + !device_link_test(dev_link, DL_FLAG_CYCLE)) + continue; + + if (__fw_devlink_relax_cycles(con_handle, + dev_link->supplier->fwnode)) { + pr_debug("%pfwf: cycle: depends on %pfwf\n", sup_handle, + dev_link->supplier->fwnode); + fw_devlink_relax_link(dev_link); + dev_link->flags |= DL_FLAG_CYCLE; + ret = true; + } + } + +out: + sup_handle->flags &= ~FWNODE_FLAG_VISITED; + put_device(sup_dev); + put_device(con_dev); + put_device(par_dev); + return ret; +} + +/** + * fw_devlink_create_devlink - Create a device link from a consumer to fwnode + * @con: consumer device for the device link + * @sup_handle: fwnode handle of supplier + * @link: fwnode link that's being converted to a device link + * + * This function will try to create a device link between the consumer device + * @con and the supplier device represented by @sup_handle. + * + * The supplier has to be provided as a fwnode because incorrect cycles in + * fwnode links can sometimes cause the supplier device to never be created. + * This function detects such cases and returns an error if it cannot create a + * device link from the consumer to a missing supplier. + * + * Returns, + * 0 on successfully creating a device link + * -EINVAL if the device link cannot be created as expected + * -EAGAIN if the device link cannot be created right now, but it may be + * possible to do that in the future + */ +static int fw_devlink_create_devlink(struct device *con, + struct fwnode_handle *sup_handle, + struct fwnode_link *link) +{ + struct device *sup_dev; + int ret = 0; + u32 flags; + + if (link->flags & FWLINK_FLAG_IGNORE) + return 0; + + /* + * In some cases, a device P might also be a supplier to its child node + * C. However, this would defer the probe of C until the probe of P + * completes successfully. This is perfectly fine in the device driver + * model. device_add() doesn't guarantee probe completion of the device + * by the time it returns. + * + * However, there are a few drivers that assume C will finish probing + * as soon as it's added and before P finishes probing. So, we provide + * a flag to let fw_devlink know not to delay the probe of C until the + * probe of P completes successfully. + * + * When such a flag is set, we can't create device links where P is the + * supplier of C as that would delay the probe of C. + */ + if (sup_handle->flags & FWNODE_FLAG_NEEDS_CHILD_BOUND_ON_ADD && + fwnode_is_ancestor_of(sup_handle, con->fwnode)) + return -EINVAL; + + /* + * Don't try to optimize by not calling the cycle detection logic under + * certain conditions. There's always some corner case that won't get + * detected. + */ + device_links_write_lock(); + if (__fw_devlink_relax_cycles(link->consumer, sup_handle)) { + __fwnode_link_cycle(link); + pr_debug("----- cycle: end -----\n"); + pr_info("%pfwf: Fixed dependency cycle(s) with %pfwf\n", + link->consumer, sup_handle); + } + device_links_write_unlock(); + + if (con->fwnode == link->consumer) + flags = fw_devlink_get_flags(link->flags); + else + flags = FW_DEVLINK_FLAGS_PERMISSIVE; + + if (sup_handle->flags & FWNODE_FLAG_NOT_DEVICE) + sup_dev = fwnode_get_next_parent_dev(sup_handle); + else + sup_dev = get_dev_from_fwnode(sup_handle); + + if (sup_dev) { + /* + * If it's one of those drivers that don't actually bind to + * their device using driver core, then don't wait on this + * supplier device indefinitely. + */ + if (sup_dev->links.status == DL_DEV_NO_DRIVER && + sup_handle->flags & FWNODE_FLAG_INITIALIZED) { + dev_dbg(con, + "Not linking %pfwf - dev might never probe\n", + sup_handle); + ret = -EINVAL; + goto out; + } + + if (con != sup_dev && !device_link_add(con, sup_dev, flags)) { + dev_err(con, "Failed to create device link (0x%x) with supplier %s for %pfwf\n", + flags, dev_name(sup_dev), link->consumer); + ret = -EINVAL; + } + + goto out; + } + + /* + * Supplier or supplier's ancestor already initialized without a struct + * device or being probed by a driver. + */ + if (fwnode_init_without_drv(sup_handle) || + fwnode_ancestor_init_without_drv(sup_handle)) { + dev_dbg(con, "Not linking %pfwf - might never become dev\n", + sup_handle); + return -EINVAL; + } + + ret = -EAGAIN; +out: + put_device(sup_dev); + return ret; +} + +/** + * __fw_devlink_link_to_consumers - Create device links to consumers of a device + * @dev: Device that needs to be linked to its consumers + * + * This function looks at all the consumer fwnodes of @dev and creates device + * links between the consumer device and @dev (supplier). + * + * If the consumer device has not been added yet, then this function creates a + * SYNC_STATE_ONLY link between @dev (supplier) and the closest ancestor device + * of the consumer fwnode. This is necessary to make sure @dev doesn't get a + * sync_state() callback before the real consumer device gets to be added and + * then probed. + * + * Once device links are created from the real consumer to @dev (supplier), the + * fwnode links are deleted. + */ +static void __fw_devlink_link_to_consumers(struct device *dev) +{ + struct fwnode_handle *fwnode = dev->fwnode; + struct fwnode_link *link, *tmp; + + list_for_each_entry_safe(link, tmp, &fwnode->consumers, s_hook) { + struct device *con_dev; + bool own_link = true; + int ret; + + con_dev = get_dev_from_fwnode(link->consumer); + /* + * If consumer device is not available yet, make a "proxy" + * SYNC_STATE_ONLY link from the consumer's parent device to + * the supplier device. This is necessary to make sure the + * supplier doesn't get a sync_state() callback before the real + * consumer can create a device link to the supplier. + * + * This proxy link step is needed to handle the case where the + * consumer's parent device is added before the supplier. + */ + if (!con_dev) { + con_dev = fwnode_get_next_parent_dev(link->consumer); + /* + * However, if the consumer's parent device is also the + * parent of the supplier, don't create a + * consumer-supplier link from the parent to its child + * device. Such a dependency is impossible. + */ + if (con_dev && + fwnode_is_ancestor_of(con_dev->fwnode, fwnode)) { + put_device(con_dev); + con_dev = NULL; + } else { + own_link = false; + } + } + + if (!con_dev) + continue; + + ret = fw_devlink_create_devlink(con_dev, fwnode, link); + put_device(con_dev); + if (!own_link || ret == -EAGAIN) + continue; + + __fwnode_link_del(link); + } +} + +/** + * __fw_devlink_link_to_suppliers - Create device links to suppliers of a device + * @dev: The consumer device that needs to be linked to its suppliers + * @fwnode: Root of the fwnode tree that is used to create device links + * + * This function looks at all the supplier fwnodes of fwnode tree rooted at + * @fwnode and creates device links between @dev (consumer) and all the + * supplier devices of the entire fwnode tree at @fwnode. + * + * The function creates normal (non-SYNC_STATE_ONLY) device links between @dev + * and the real suppliers of @dev. Once these device links are created, the + * fwnode links are deleted. + * + * In addition, it also looks at all the suppliers of the entire fwnode tree + * because some of the child devices of @dev that have not been added yet + * (because @dev hasn't probed) might already have their suppliers added to + * driver core. So, this function creates SYNC_STATE_ONLY device links between + * @dev (consumer) and these suppliers to make sure they don't execute their + * sync_state() callbacks before these child devices have a chance to create + * their device links. The fwnode links that correspond to the child devices + * aren't delete because they are needed later to create the device links + * between the real consumer and supplier devices. + */ +static void __fw_devlink_link_to_suppliers(struct device *dev, + struct fwnode_handle *fwnode) +{ + bool own_link = (dev->fwnode == fwnode); + struct fwnode_link *link, *tmp; + struct fwnode_handle *child = NULL; + + list_for_each_entry_safe(link, tmp, &fwnode->suppliers, c_hook) { + int ret; + struct fwnode_handle *sup = link->supplier; + + ret = fw_devlink_create_devlink(dev, sup, link); + if (!own_link || ret == -EAGAIN) + continue; + + __fwnode_link_del(link); + } + + /* + * Make "proxy" SYNC_STATE_ONLY device links to represent the needs of + * all the descendants. This proxy link step is needed to handle the + * case where the supplier is added before the consumer's parent device + * (@dev). + */ + while ((child = fwnode_get_next_available_child_node(fwnode, child))) + __fw_devlink_link_to_suppliers(dev, child); +} + +static void fw_devlink_link_device(struct device *dev) +{ + struct fwnode_handle *fwnode = dev->fwnode; + + if (!fw_devlink_flags) + return; + + fw_devlink_parse_fwtree(fwnode); + + guard(mutex)(&fwnode_link_lock); + + __fw_devlink_link_to_consumers(dev); + __fw_devlink_link_to_suppliers(dev, fwnode); +} + +/* Device links support end. */ -int (*platform_notify)(struct device *dev) = NULL; -int (*platform_notify_remove)(struct device *dev) = NULL; static struct kobject *dev_kobj; -struct kobject *sysfs_dev_char_kobj; -struct kobject *sysfs_dev_block_kobj; + +/* /sys/dev/char */ +static struct kobject *sysfs_dev_char_kobj; + +/* /sys/dev/block */ +static struct kobject *sysfs_dev_block_kobj; + +static DEFINE_MUTEX(device_hotplug_lock); + +void lock_device_hotplug(void) +{ + mutex_lock(&device_hotplug_lock); +} + +void unlock_device_hotplug(void) +{ + mutex_unlock(&device_hotplug_lock); +} + +int lock_device_hotplug_sysfs(void) +{ + if (mutex_trylock(&device_hotplug_lock)) + return 0; + + /* Avoid busy looping (5 ms of sleep should do). */ + msleep(5); + return restart_syscall(); +} #ifdef CONFIG_BLOCK static inline int device_is_not_partition(struct device *dev) @@ -61,6 +2372,20 @@ static inline int device_is_not_partition(struct device *dev) } #endif +static void device_platform_notify(struct device *dev) +{ + acpi_device_notify(dev); + + software_node_notify(dev); +} + +static void device_platform_notify_remove(struct device *dev) +{ + software_node_notify_remove(dev); + + acpi_device_notify_remove(dev); +} + /** * dev_driver_string - Return a device's driver name, if at all possible * @dev: struct device to get the name of @@ -78,10 +2403,8 @@ const char *dev_driver_string(const struct device *dev) * so be careful about accessing it. dev->bus and dev->class should * never change once they are set, so they don't need special care. */ - drv = ACCESS_ONCE(dev->driver); - return drv ? drv->name : - (dev->bus ? dev->bus->name : - (dev->class ? dev->class->name : "")); + drv = READ_ONCE(dev->driver); + return drv ? drv->name : dev_bus_name(dev); } EXPORT_SYMBOL(dev_driver_string); @@ -97,8 +2420,8 @@ static ssize_t dev_attr_show(struct kobject *kobj, struct attribute *attr, if (dev_attr->show) ret = dev_attr->show(dev, dev_attr, buf); if (ret >= (ssize_t)PAGE_SIZE) { - print_symbol("dev_attr_show: %s returned bad count\n", - (unsigned long)dev_attr->show); + printk("dev_attr_show: %pS returned bad count\n", + dev_attr->show); } return ret; } @@ -127,10 +2450,12 @@ ssize_t device_store_ulong(struct device *dev, const char *buf, size_t size) { struct dev_ext_attribute *ea = to_ext_attr(attr); - char *end; - unsigned long new = simple_strtoul(buf, &end, 0); - if (end == buf) - return -EINVAL; + int ret; + unsigned long new; + + ret = kstrtoul(buf, 0, &new); + if (ret) + return ret; *(unsigned long *)(ea->var) = new; /* Always return full write size even if we didn't consume all */ return size; @@ -142,7 +2467,7 @@ ssize_t device_show_ulong(struct device *dev, char *buf) { struct dev_ext_attribute *ea = to_ext_attr(attr); - return snprintf(buf, PAGE_SIZE, "%lx\n", *(unsigned long *)(ea->var)); + return sysfs_emit(buf, "%lx\n", *(unsigned long *)(ea->var)); } EXPORT_SYMBOL_GPL(device_show_ulong); @@ -151,9 +2476,14 @@ ssize_t device_store_int(struct device *dev, const char *buf, size_t size) { struct dev_ext_attribute *ea = to_ext_attr(attr); - char *end; - long new = simple_strtol(buf, &end, 0); - if (end == buf || new > INT_MAX || new < INT_MIN) + int ret; + long new; + + ret = kstrtol(buf, 0, &new); + if (ret) + return ret; + + if (new > INT_MAX || new < INT_MIN) return -EINVAL; *(int *)(ea->var) = new; /* Always return full write size even if we didn't consume all */ @@ -167,7 +2497,7 @@ ssize_t device_show_int(struct device *dev, { struct dev_ext_attribute *ea = to_ext_attr(attr); - return snprintf(buf, PAGE_SIZE, "%d\n", *(int *)(ea->var)); + return sysfs_emit(buf, "%d\n", *(int *)(ea->var)); } EXPORT_SYMBOL_GPL(device_show_int); @@ -176,7 +2506,7 @@ ssize_t device_store_bool(struct device *dev, struct device_attribute *attr, { struct dev_ext_attribute *ea = to_ext_attr(attr); - if (strtobool(buf, ea->var) < 0) + if (kstrtobool(buf, ea->var) < 0) return -EINVAL; return size; @@ -188,10 +2518,19 @@ ssize_t device_show_bool(struct device *dev, struct device_attribute *attr, { struct dev_ext_attribute *ea = to_ext_attr(attr); - return snprintf(buf, PAGE_SIZE, "%d\n", *(bool *)(ea->var)); + return sysfs_emit(buf, "%d\n", *(bool *)(ea->var)); } EXPORT_SYMBOL_GPL(device_show_bool); +ssize_t device_show_string(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct dev_ext_attribute *ea = to_ext_attr(attr); + + return sysfs_emit(buf, "%s\n", (char *)ea->var); +} +EXPORT_SYMBOL_GPL(device_show_string); + /** * device_release - free device structure. * @kobj: device's kobject. @@ -216,6 +2555,8 @@ static void device_release(struct kobject *kobj) */ devres_release_all(dev); + kfree(dev->dma_range_map); + if (dev->release) dev->release(dev); else if (dev->type && dev->type->release) @@ -223,36 +2564,44 @@ static void device_release(struct kobject *kobj) else if (dev->class && dev->class->dev_release) dev->class->dev_release(dev); else - WARN(1, KERN_ERR "Device '%s' does not have a release() " - "function, it is broken and must be fixed.\n", + WARN(1, KERN_ERR "Device '%s' does not have a release() function, it is broken and must be fixed. See Documentation/core-api/kobject.rst.\n", dev_name(dev)); kfree(p); } -static const void *device_namespace(struct kobject *kobj) +static const void *device_namespace(const struct kobject *kobj) { - struct device *dev = kobj_to_dev(kobj); + const struct device *dev = kobj_to_dev(kobj); const void *ns = NULL; - if (dev->class && dev->class->ns_type) + if (dev->class && dev->class->namespace) ns = dev->class->namespace(dev); return ns; } -static struct kobj_type device_ktype = { +static void device_get_ownership(const struct kobject *kobj, kuid_t *uid, kgid_t *gid) +{ + const struct device *dev = kobj_to_dev(kobj); + + if (dev->class && dev->class->get_ownership) + dev->class->get_ownership(dev, uid, gid); +} + +static const struct kobj_type device_ktype = { .release = device_release, .sysfs_ops = &dev_sysfs_ops, .namespace = device_namespace, + .get_ownership = device_get_ownership, }; -static int dev_uevent_filter(struct kset *kset, struct kobject *kobj) +static int dev_uevent_filter(const struct kobject *kobj) { - struct kobj_type *ktype = get_ktype(kobj); + const struct kobj_type *ktype = get_ktype(kobj); if (ktype == &device_ktype) { - struct device *dev = kobj_to_dev(kobj); + const struct device *dev = kobj_to_dev(kobj); if (dev->bus) return 1; if (dev->class) @@ -261,9 +2610,9 @@ static int dev_uevent_filter(struct kset *kset, struct kobject *kobj) return 0; } -static const char *dev_uevent_name(struct kset *kset, struct kobject *kobj) +static const char *dev_uevent_name(const struct kobject *kobj) { - struct device *dev = kobj_to_dev(kobj); + const struct device *dev = kobj_to_dev(kobj); if (dev->bus) return dev->bus->name; @@ -272,10 +2621,38 @@ static const char *dev_uevent_name(struct kset *kset, struct kobject *kobj) return NULL; } -static int dev_uevent(struct kset *kset, struct kobject *kobj, - struct kobj_uevent_env *env) +/* + * Try filling "DRIVER=<name>" uevent variable for a device. Because this + * function may race with binding and unbinding the device from a driver, + * we need to be careful. Binding is generally safe, at worst we miss the + * fact that the device is already bound to a driver (but the driver + * information that is delivered through uevents is best-effort, it may + * become obsolete as soon as it is generated anyways). Unbinding is more + * risky as driver pointer is transitioning to NULL, so READ_ONCE() should + * be used to make sure we are dealing with the same pointer, and to + * ensure that driver structure is not going to disappear from under us + * we take bus' drivers klist lock. The assumption that only registered + * driver can be bound to a device, and to unregister a driver bus code + * will take the same lock. + */ +static void dev_driver_uevent(const struct device *dev, struct kobj_uevent_env *env) { - struct device *dev = kobj_to_dev(kobj); + struct subsys_private *sp = bus_to_subsys(dev->bus); + + if (sp) { + scoped_guard(spinlock, &sp->klist_drivers.k_lock) { + struct device_driver *drv = READ_ONCE(dev->driver); + if (drv) + add_uevent_var(env, "DRIVER=%s", drv->name); + } + + subsys_put(sp); + } +} + +static int dev_uevent(const struct kobject *kobj, struct kobj_uevent_env *env) +{ + const struct device *dev = kobj_to_dev(kobj); int retval = 0; /* add device node properties if present */ @@ -304,8 +2681,8 @@ static int dev_uevent(struct kset *kset, struct kobject *kobj, if (dev->type && dev->type->name) add_uevent_var(env, "DEVTYPE=%s", dev->type->name); - if (dev->driver) - add_uevent_var(env, "DRIVER=%s", dev->driver->name); + /* Add "DRIVER=%s" variable if the device is bound to a driver */ + dev_driver_uevent(dev, env); /* Add common DT information about the device */ of_device_uevent(dev, env); @@ -345,14 +2722,14 @@ static const struct kset_uevent_ops device_uevent_ops = { .uevent = dev_uevent, }; -static ssize_t show_uevent(struct device *dev, struct device_attribute *attr, +static ssize_t uevent_show(struct device *dev, struct device_attribute *attr, char *buf) { struct kobject *top_kobj; struct kset *kset; struct kobj_uevent_env *env = NULL; int i; - size_t count = 0; + int len = 0; int retval; /* search the kset, the device belongs to */ @@ -368,7 +2745,7 @@ static ssize_t show_uevent(struct device *dev, struct device_attribute *attr, /* respect filter */ if (kset->uevent_ops && kset->uevent_ops->filter) - if (!kset->uevent_ops->filter(kset, &dev->kobj)) + if (!kset->uevent_ops->filter(&dev->kobj)) goto out; env = kzalloc(sizeof(struct kobj_uevent_env), GFP_KERNEL); @@ -376,170 +2753,159 @@ static ssize_t show_uevent(struct device *dev, struct device_attribute *attr, return -ENOMEM; /* let the kset specific function add its keys */ - retval = kset->uevent_ops->uevent(kset, &dev->kobj, env); + retval = kset->uevent_ops->uevent(&dev->kobj, env); if (retval) goto out; /* copy keys to file */ for (i = 0; i < env->envp_idx; i++) - count += sprintf(&buf[count], "%s\n", env->envp[i]); + len += sysfs_emit_at(buf, len, "%s\n", env->envp[i]); out: kfree(env); - return count; + return len; } -static ssize_t store_uevent(struct device *dev, struct device_attribute *attr, +static ssize_t uevent_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - enum kobject_action action; + int rc; + + rc = kobject_synth_uevent(&dev->kobj, buf, count); + + if (rc) { + dev_err(dev, "uevent: failed to send synthetic uevent: %d\n", rc); + return rc; + } - if (kobject_action_type(buf, count, &action) == 0) - kobject_uevent(&dev->kobj, action); - else - dev_err(dev, "uevent: unknown action-string\n"); return count; } +static DEVICE_ATTR_RW(uevent); -static struct device_attribute uevent_attr = - __ATTR(uevent, S_IRUGO | S_IWUSR, show_uevent, store_uevent); - -static ssize_t show_online(struct device *dev, struct device_attribute *attr, +static ssize_t online_show(struct device *dev, struct device_attribute *attr, char *buf) { bool val; - lock_device_hotplug(); + device_lock(dev); val = !dev->offline; - unlock_device_hotplug(); - return sprintf(buf, "%u\n", val); + device_unlock(dev); + return sysfs_emit(buf, "%u\n", val); } -static ssize_t store_online(struct device *dev, struct device_attribute *attr, +static ssize_t online_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { bool val; int ret; - ret = strtobool(buf, &val); + ret = kstrtobool(buf, &val); if (ret < 0) return ret; - lock_device_hotplug(); + ret = lock_device_hotplug_sysfs(); + if (ret) + return ret; + ret = val ? device_online(dev) : device_offline(dev); unlock_device_hotplug(); return ret < 0 ? ret : count; } +static DEVICE_ATTR_RW(online); -static struct device_attribute online_attr = - __ATTR(online, S_IRUGO | S_IWUSR, show_online, store_online); - -static int device_add_attributes(struct device *dev, - struct device_attribute *attrs) +static ssize_t removable_show(struct device *dev, struct device_attribute *attr, + char *buf) { - int error = 0; - int i; + const char *loc; - if (attrs) { - for (i = 0; attr_name(attrs[i]); i++) { - error = device_create_file(dev, &attrs[i]); - if (error) - break; - } - if (error) - while (--i >= 0) - device_remove_file(dev, &attrs[i]); + switch (dev->removable) { + case DEVICE_REMOVABLE: + loc = "removable"; + break; + case DEVICE_FIXED: + loc = "fixed"; + break; + default: + loc = "unknown"; } - return error; + return sysfs_emit(buf, "%s\n", loc); } +static DEVICE_ATTR_RO(removable); -static void device_remove_attributes(struct device *dev, - struct device_attribute *attrs) +int device_add_groups(struct device *dev, const struct attribute_group **groups) { - int i; - - if (attrs) - for (i = 0; attr_name(attrs[i]); i++) - device_remove_file(dev, &attrs[i]); + return sysfs_create_groups(&dev->kobj, groups); } +EXPORT_SYMBOL_GPL(device_add_groups); -static int device_add_bin_attributes(struct device *dev, - struct bin_attribute *attrs) +void device_remove_groups(struct device *dev, + const struct attribute_group **groups) { - int error = 0; - int i; - - if (attrs) { - for (i = 0; attr_name(attrs[i]); i++) { - error = device_create_bin_file(dev, &attrs[i]); - if (error) - break; - } - if (error) - while (--i >= 0) - device_remove_bin_file(dev, &attrs[i]); - } - return error; + sysfs_remove_groups(&dev->kobj, groups); } +EXPORT_SYMBOL_GPL(device_remove_groups); -static void device_remove_bin_attributes(struct device *dev, - struct bin_attribute *attrs) +union device_attr_group_devres { + const struct attribute_group *group; + const struct attribute_group **groups; +}; + +static void devm_attr_group_remove(struct device *dev, void *res) { - int i; + union device_attr_group_devres *devres = res; + const struct attribute_group *group = devres->group; - if (attrs) - for (i = 0; attr_name(attrs[i]); i++) - device_remove_bin_file(dev, &attrs[i]); + dev_dbg(dev, "%s: removing group %p\n", __func__, group); + sysfs_remove_group(&dev->kobj, group); } -static int device_add_groups(struct device *dev, - const struct attribute_group **groups) +/** + * devm_device_add_group - given a device, create a managed attribute group + * @dev: The device to create the group for + * @grp: The attribute group to create + * + * This function creates a group for the first time. It will explicitly + * warn and error if any of the attribute files being created already exist. + * + * Returns 0 on success or error code on failure. + */ +int devm_device_add_group(struct device *dev, const struct attribute_group *grp) { - int error = 0; - int i; + union device_attr_group_devres *devres; + int error; - if (groups) { - for (i = 0; groups[i]; i++) { - error = sysfs_create_group(&dev->kobj, groups[i]); - if (error) { - while (--i >= 0) - sysfs_remove_group(&dev->kobj, - groups[i]); - break; - } - } - } - return error; -} + devres = devres_alloc(devm_attr_group_remove, + sizeof(*devres), GFP_KERNEL); + if (!devres) + return -ENOMEM; -static void device_remove_groups(struct device *dev, - const struct attribute_group **groups) -{ - int i; + error = sysfs_create_group(&dev->kobj, grp); + if (error) { + devres_free(devres); + return error; + } - if (groups) - for (i = 0; groups[i]; i++) - sysfs_remove_group(&dev->kobj, groups[i]); + devres->group = grp; + devres_add(dev, devres); + return 0; } +EXPORT_SYMBOL_GPL(devm_device_add_group); static int device_add_attrs(struct device *dev) { - struct class *class = dev->class; + const struct class *class = dev->class; const struct device_type *type = dev->type; int error; if (class) { - error = device_add_attributes(dev, class->dev_attrs); + error = device_add_groups(dev, class->dev_groups); if (error) return error; - error = device_add_bin_attributes(dev, class->dev_bin_attrs); - if (error) - goto err_remove_class_attrs; } if (type) { error = device_add_groups(dev, type->groups); if (error) - goto err_remove_class_bin_attrs; + goto err_remove_class_groups; } error = device_add_groups(dev, dev->groups); @@ -547,57 +2913,129 @@ static int device_add_attrs(struct device *dev) goto err_remove_type_groups; if (device_supports_offline(dev) && !dev->offline_disabled) { - error = device_create_file(dev, &online_attr); + error = device_create_file(dev, &dev_attr_online); + if (error) + goto err_remove_dev_groups; + } + + if (fw_devlink_flags && !fw_devlink_is_permissive() && dev->fwnode) { + error = device_create_file(dev, &dev_attr_waiting_for_supplier); + if (error) + goto err_remove_dev_online; + } + + if (dev_removable_is_valid(dev)) { + error = device_create_file(dev, &dev_attr_removable); if (error) - goto err_remove_type_groups; + goto err_remove_dev_waiting_for_supplier; + } + + if (dev_add_physical_location(dev)) { + error = device_add_group(dev, + &dev_attr_physical_location_group); + if (error) + goto err_remove_dev_removable; } return 0; + err_remove_dev_removable: + device_remove_file(dev, &dev_attr_removable); + err_remove_dev_waiting_for_supplier: + device_remove_file(dev, &dev_attr_waiting_for_supplier); + err_remove_dev_online: + device_remove_file(dev, &dev_attr_online); + err_remove_dev_groups: + device_remove_groups(dev, dev->groups); err_remove_type_groups: if (type) device_remove_groups(dev, type->groups); - err_remove_class_bin_attrs: + err_remove_class_groups: if (class) - device_remove_bin_attributes(dev, class->dev_bin_attrs); - err_remove_class_attrs: - if (class) - device_remove_attributes(dev, class->dev_attrs); + device_remove_groups(dev, class->dev_groups); return error; } static void device_remove_attrs(struct device *dev) { - struct class *class = dev->class; + const struct class *class = dev->class; const struct device_type *type = dev->type; - device_remove_file(dev, &online_attr); + if (dev->physical_location) { + device_remove_group(dev, &dev_attr_physical_location_group); + kfree(dev->physical_location); + } + + device_remove_file(dev, &dev_attr_removable); + device_remove_file(dev, &dev_attr_waiting_for_supplier); + device_remove_file(dev, &dev_attr_online); device_remove_groups(dev, dev->groups); if (type) device_remove_groups(dev, type->groups); - if (class) { - device_remove_attributes(dev, class->dev_attrs); - device_remove_bin_attributes(dev, class->dev_bin_attrs); - } + if (class) + device_remove_groups(dev, class->dev_groups); } - -static ssize_t show_dev(struct device *dev, struct device_attribute *attr, +static ssize_t dev_show(struct device *dev, struct device_attribute *attr, char *buf) { return print_dev_t(buf, dev->devt); } - -static struct device_attribute devt_attr = - __ATTR(dev, S_IRUGO, show_dev, NULL); +static DEVICE_ATTR_RO(dev); /* /sys/devices/ */ struct kset *devices_kset; /** + * devices_kset_move_before - Move device in the devices_kset's list. + * @deva: Device to move. + * @devb: Device @deva should come before. + */ +static void devices_kset_move_before(struct device *deva, struct device *devb) +{ + if (!devices_kset) + return; + pr_debug("devices_kset: Moving %s before %s\n", + dev_name(deva), dev_name(devb)); + spin_lock(&devices_kset->list_lock); + list_move_tail(&deva->kobj.entry, &devb->kobj.entry); + spin_unlock(&devices_kset->list_lock); +} + +/** + * devices_kset_move_after - Move device in the devices_kset's list. + * @deva: Device to move + * @devb: Device @deva should come after. + */ +static void devices_kset_move_after(struct device *deva, struct device *devb) +{ + if (!devices_kset) + return; + pr_debug("devices_kset: Moving %s after %s\n", + dev_name(deva), dev_name(devb)); + spin_lock(&devices_kset->list_lock); + list_move(&deva->kobj.entry, &devb->kobj.entry); + spin_unlock(&devices_kset->list_lock); +} + +/** + * devices_kset_move_last - move the device to the end of devices_kset's list. + * @dev: device to move + */ +void devices_kset_move_last(struct device *dev) +{ + if (!devices_kset) + return; + pr_debug("devices_kset: Moving %s to end of list\n", dev_name(dev)); + spin_lock(&devices_kset->list_lock); + list_move_tail(&dev->kobj.entry, &devices_kset->list); + spin_unlock(&devices_kset->list_lock); +} + +/** * device_create_file - create sysfs attribute file for device. * @dev: device. * @attr: device attribute descriptor. @@ -619,6 +3057,7 @@ int device_create_file(struct device *dev, return error; } +EXPORT_SYMBOL_GPL(device_create_file); /** * device_remove_file - remove sysfs attribute file. @@ -631,6 +3070,24 @@ void device_remove_file(struct device *dev, if (dev) sysfs_remove_file(&dev->kobj, &attr->attr); } +EXPORT_SYMBOL_GPL(device_remove_file); + +/** + * device_remove_file_self - remove sysfs attribute file from its own method. + * @dev: device. + * @attr: device attribute descriptor. + * + * See kernfs_remove_self() for details. + */ +bool device_remove_file_self(struct device *dev, + const struct device_attribute *attr) +{ + if (dev) + return sysfs_remove_file_self(&dev->kobj, &attr->attr); + else + return false; +} +EXPORT_SYMBOL_GPL(device_remove_file_self); /** * device_create_bin_file - create sysfs binary attribute file for device. @@ -660,39 +3117,6 @@ void device_remove_bin_file(struct device *dev, } EXPORT_SYMBOL_GPL(device_remove_bin_file); -/** - * device_schedule_callback_owner - helper to schedule a callback for a device - * @dev: device. - * @func: callback function to invoke later. - * @owner: module owning the callback routine - * - * Attribute methods must not unregister themselves or their parent device - * (which would amount to the same thing). Attempts to do so will deadlock, - * since unregistration is mutually exclusive with driver callbacks. - * - * Instead methods can call this routine, which will attempt to allocate - * and schedule a workqueue request to call back @func with @dev as its - * argument in the workqueue's process context. @dev will be pinned until - * @func returns. - * - * This routine is usually called via the inline device_schedule_callback(), - * which automatically sets @owner to THIS_MODULE. - * - * Returns 0 if the request was submitted, -ENOMEM if storage could not - * be allocated, -ENODEV if a reference to @owner isn't available. - * - * NOTE: This routine won't work if CONFIG_SYSFS isn't set! It uses an - * underlying sysfs routine (since it is intended for use by attribute - * methods), and if sysfs isn't available you'll get nothing but -ENOSYS. - */ -int device_schedule_callback_owner(struct device *dev, - void (*func)(struct device *), struct module *owner) -{ - return sysfs_schedule_callback(&dev->kobj, - (void (*)(void *)) func, dev, owner); -} -EXPORT_SYMBOL_GPL(device_schedule_callback_owner); - static void klist_children_get(struct klist_node *n) { struct device_private *p = to_device_private_parent(n); @@ -739,10 +3163,21 @@ void device_initialize(struct device *dev) spin_lock_init(&dev->devres_lock); INIT_LIST_HEAD(&dev->devres_head); device_pm_init(dev); - set_dev_node(dev, -1); + set_dev_node(dev, NUMA_NO_NODE); + INIT_LIST_HEAD(&dev->links.consumers); + INIT_LIST_HEAD(&dev->links.suppliers); + INIT_LIST_HEAD(&dev->links.defer_sync); + dev->links.status = DL_DEV_NO_DRIVER; +#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \ + defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \ + defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL) + dev->dma_coherent = dma_default_coherent; +#endif + swiotlb_dev_init(dev); } +EXPORT_SYMBOL_GPL(device_initialize); -struct kobject *virtual_device_parent(struct device *dev) +struct kobject *virtual_device_parent(void) { static struct kobject *virtual_dir = NULL; @@ -755,7 +3190,7 @@ struct kobject *virtual_device_parent(struct device *dev) struct class_dir { struct kobject kobj; - struct class *class; + const struct class *class; }; #define to_class_dir(obj) container_of(obj, struct class_dir, kobj) @@ -767,130 +3202,240 @@ static void class_dir_release(struct kobject *kobj) } static const -struct kobj_ns_type_operations *class_dir_child_ns_type(struct kobject *kobj) +struct kobj_ns_type_operations *class_dir_child_ns_type(const struct kobject *kobj) { - struct class_dir *dir = to_class_dir(kobj); + const struct class_dir *dir = to_class_dir(kobj); return dir->class->ns_type; } -static struct kobj_type class_dir_ktype = { +static const struct kobj_type class_dir_ktype = { .release = class_dir_release, .sysfs_ops = &kobj_sysfs_ops, .child_ns_type = class_dir_child_ns_type }; -static struct kobject * -class_dir_create_and_add(struct class *class, struct kobject *parent_kobj) +static struct kobject *class_dir_create_and_add(struct subsys_private *sp, + struct kobject *parent_kobj) { struct class_dir *dir; int retval; dir = kzalloc(sizeof(*dir), GFP_KERNEL); if (!dir) - return NULL; + return ERR_PTR(-ENOMEM); - dir->class = class; + dir->class = sp->class; kobject_init(&dir->kobj, &class_dir_ktype); - dir->kobj.kset = &class->p->glue_dirs; + dir->kobj.kset = &sp->glue_dirs; - retval = kobject_add(&dir->kobj, parent_kobj, "%s", class->name); + retval = kobject_add(&dir->kobj, parent_kobj, "%s", sp->class->name); if (retval < 0) { kobject_put(&dir->kobj); - return NULL; + return ERR_PTR(retval); } return &dir->kobj; } +static DEFINE_MUTEX(gdp_mutex); static struct kobject *get_device_parent(struct device *dev, struct device *parent) { - if (dev->class) { - static DEFINE_MUTEX(gdp_mutex); - struct kobject *kobj = NULL; + struct subsys_private *sp = class_to_subsys(dev->class); + struct kobject *kobj = NULL; + + if (sp) { struct kobject *parent_kobj; struct kobject *k; -#ifdef CONFIG_BLOCK - /* block disks show up in /sys/block */ - if (sysfs_deprecated && dev->class == &block_class) { - if (parent && parent->class == &block_class) - return &parent->kobj; - return &block_class.p->subsys.kobj; - } -#endif - /* * If we have no parent, we live in "virtual". * Class-devices with a non class-device as parent, live * in a "glue" directory to prevent namespace collisions. */ if (parent == NULL) - parent_kobj = virtual_device_parent(dev); - else if (parent->class && !dev->class->ns_type) + parent_kobj = virtual_device_parent(); + else if (parent->class && !dev->class->ns_type) { + subsys_put(sp); return &parent->kobj; - else + } else { parent_kobj = &parent->kobj; + } mutex_lock(&gdp_mutex); /* find our class-directory at the parent and reference it */ - spin_lock(&dev->class->p->glue_dirs.list_lock); - list_for_each_entry(k, &dev->class->p->glue_dirs.list, entry) + spin_lock(&sp->glue_dirs.list_lock); + list_for_each_entry(k, &sp->glue_dirs.list, entry) if (k->parent == parent_kobj) { kobj = kobject_get(k); break; } - spin_unlock(&dev->class->p->glue_dirs.list_lock); + spin_unlock(&sp->glue_dirs.list_lock); if (kobj) { mutex_unlock(&gdp_mutex); + subsys_put(sp); return kobj; } /* or create a new class-directory at the parent device */ - k = class_dir_create_and_add(dev->class, parent_kobj); + k = class_dir_create_and_add(sp, parent_kobj); /* do not emit an uevent for this simple "glue" directory */ mutex_unlock(&gdp_mutex); + subsys_put(sp); return k; } /* subsystems can specify a default root directory for their devices */ - if (!parent && dev->bus && dev->bus->dev_root) - return &dev->bus->dev_root->kobj; + if (!parent && dev->bus) { + struct device *dev_root = bus_get_dev_root(dev->bus); + + if (dev_root) { + kobj = &dev_root->kobj; + put_device(dev_root); + return kobj; + } + } if (parent) return &parent->kobj; return NULL; } +static inline bool live_in_glue_dir(struct kobject *kobj, + struct device *dev) +{ + struct subsys_private *sp; + bool retval; + + if (!kobj || !dev->class) + return false; + + sp = class_to_subsys(dev->class); + if (!sp) + return false; + + if (kobj->kset == &sp->glue_dirs) + retval = true; + else + retval = false; + + subsys_put(sp); + return retval; +} + +static inline struct kobject *get_glue_dir(struct device *dev) +{ + return dev->kobj.parent; +} + +/** + * kobject_has_children - Returns whether a kobject has children. + * @kobj: the object to test + * + * This will return whether a kobject has other kobjects as children. + * + * It does NOT account for the presence of attribute files, only sub + * directories. It also assumes there is no concurrent addition or + * removal of such children, and thus relies on external locking. + */ +static inline bool kobject_has_children(struct kobject *kobj) +{ + WARN_ON_ONCE(kref_read(&kobj->kref) == 0); + + return kobj->sd && kobj->sd->dir.subdirs; +} + +/* + * make sure cleaning up dir as the last step, we need to make + * sure .release handler of kobject is run with holding the + * global lock + */ static void cleanup_glue_dir(struct device *dev, struct kobject *glue_dir) { + unsigned int ref; + /* see if we live in a "glue" directory */ - if (!glue_dir || !dev->class || - glue_dir->kset != &dev->class->p->glue_dirs) + if (!live_in_glue_dir(glue_dir, dev)) return; + mutex_lock(&gdp_mutex); + /** + * There is a race condition between removing glue directory + * and adding a new device under the glue directory. + * + * CPU1: CPU2: + * + * device_add() + * get_device_parent() + * class_dir_create_and_add() + * kobject_add_internal() + * create_dir() // create glue_dir + * + * device_add() + * get_device_parent() + * kobject_get() // get glue_dir + * + * device_del() + * cleanup_glue_dir() + * kobject_del(glue_dir) + * + * kobject_add() + * kobject_add_internal() + * create_dir() // in glue_dir + * sysfs_create_dir_ns() + * kernfs_create_dir_ns(sd) + * + * sysfs_remove_dir() // glue_dir->sd=NULL + * sysfs_put() // free glue_dir->sd + * + * // sd is freed + * kernfs_new_node(sd) + * kernfs_get(glue_dir) + * kernfs_add_one() + * kernfs_put() + * + * Before CPU1 remove last child device under glue dir, if CPU2 add + * a new device under glue dir, the glue_dir kobject reference count + * will be increase to 2 in kobject_get(k). And CPU2 has been called + * kernfs_create_dir_ns(). Meanwhile, CPU1 call sysfs_remove_dir() + * and sysfs_put(). This result in glue_dir->sd is freed. + * + * Then the CPU2 will see a stale "empty" but still potentially used + * glue dir around in kernfs_new_node(). + * + * In order to avoid this happening, we also should make sure that + * kernfs_node for glue_dir is released in CPU1 only when refcount + * for glue_dir kobj is 1. + */ + ref = kref_read(&glue_dir->kref); + if (!kobject_has_children(glue_dir) && !--ref) + kobject_del(glue_dir); kobject_put(glue_dir); -} - -static void cleanup_device_parent(struct device *dev) -{ - cleanup_glue_dir(dev, dev->kobj.parent); + mutex_unlock(&gdp_mutex); } static int device_add_class_symlinks(struct device *dev) { + struct device_node *of_node = dev_of_node(dev); + struct subsys_private *sp; int error; - if (!dev->class) + if (of_node) { + error = sysfs_create_link(&dev->kobj, of_node_kobj(of_node), "of_node"); + if (error) + dev_warn(dev, "Error %d creating of_node link\n",error); + /* An error here doesn't warrant bringing down the device */ + } + + sp = class_to_subsys(dev->class); + if (!sp) return 0; - error = sysfs_create_link(&dev->kobj, - &dev->class->p->subsys.kobj, - "subsystem"); + error = sysfs_create_link(&dev->kobj, &sp->subsys.kobj, "subsystem"); if (error) - goto out; + goto out_devnode; if (dev->parent && device_is_not_partition(dev)) { error = sysfs_create_link(&dev->kobj, &dev->parent->kobj, @@ -899,42 +3444,38 @@ static int device_add_class_symlinks(struct device *dev) goto out_subsys; } -#ifdef CONFIG_BLOCK - /* /sys/block has directories and does not need symlinks */ - if (sysfs_deprecated && dev->class == &block_class) - return 0; -#endif - /* link in the class directory pointing to the device */ - error = sysfs_create_link(&dev->class->p->subsys.kobj, - &dev->kobj, dev_name(dev)); + error = sysfs_create_link(&sp->subsys.kobj, &dev->kobj, dev_name(dev)); if (error) goto out_device; - - return 0; + goto exit; out_device: sysfs_remove_link(&dev->kobj, "device"); - out_subsys: sysfs_remove_link(&dev->kobj, "subsystem"); -out: +out_devnode: + sysfs_remove_link(&dev->kobj, "of_node"); +exit: + subsys_put(sp); return error; } static void device_remove_class_symlinks(struct device *dev) { - if (!dev->class) + struct subsys_private *sp = class_to_subsys(dev->class); + + if (dev_of_node(dev)) + sysfs_remove_link(&dev->kobj, "of_node"); + + if (!sp) return; if (dev->parent && device_is_not_partition(dev)) sysfs_remove_link(&dev->kobj, "device"); sysfs_remove_link(&dev->kobj, "subsystem"); -#ifdef CONFIG_BLOCK - if (sysfs_deprecated && dev->class == &block_class) - return; -#endif - sysfs_delete_link(&dev->class->p->subsys.kobj, &dev->kobj, dev_name(dev)); + sysfs_delete_link(&sp->subsys.kobj, &dev->kobj, dev_name(dev)); + subsys_put(sp); } /** @@ -954,27 +3495,13 @@ int dev_set_name(struct device *dev, const char *fmt, ...) } EXPORT_SYMBOL_GPL(dev_set_name); -/** - * device_to_dev_kobj - select a /sys/dev/ directory for the device - * @dev: device - * - * By default we select char/ for new entries. Setting class->dev_obj - * to NULL prevents an entry from being created. class->dev_kobj must - * be set (or cleared) before any devices are registered to the class - * otherwise device_create_sys_dev_entry() and - * device_remove_sys_dev_entry() will disagree about the presence of - * the link. - */ +/* select a /sys/dev/ directory for the device */ static struct kobject *device_to_dev_kobj(struct device *dev) { - struct kobject *kobj; - - if (dev->class) - kobj = dev->class->dev_kobj; + if (is_blockdev(dev)) + return sysfs_dev_block_kobj; else - kobj = sysfs_dev_char_kobj; - - return kobj; + return sysfs_dev_char_kobj; } static int device_create_sys_dev_entry(struct device *dev) @@ -1002,7 +3529,7 @@ static void device_remove_sys_dev_entry(struct device *dev) } } -int device_private_init(struct device *dev) +static int device_private_init(struct device *dev) { dev->p = kzalloc(sizeof(*dev->p), GFP_KERNEL); if (!dev->p) @@ -1035,13 +3562,20 @@ int device_private_init(struct device *dev) * NOTE: _Never_ directly free @dev after calling this function, even * if it returned an error! Always use put_device() to give up your * reference instead. + * + * Rule of thumb is: if device_add() succeeds, you should call + * device_del() when you want to get rid of it. If device_add() has + * *not* succeeded, use *only* put_device() to drop the reference + * count. */ int device_add(struct device *dev) { - struct device *parent = NULL; + struct subsys_private *sp; + struct device *parent; struct kobject *kobj; struct class_interface *class_intf; int error = -EINVAL; + struct kobject *glue_dir = NULL; dev = get_device(dev); if (!dev) @@ -1059,56 +3593,50 @@ int device_add(struct device *dev) * the name, and force the use of dev_name() */ if (dev->init_name) { - dev_set_name(dev, "%s", dev->init_name); + error = dev_set_name(dev, "%s", dev->init_name); dev->init_name = NULL; } + if (dev_name(dev)) + error = 0; /* subsystems can specify simple device enumeration */ - if (!dev_name(dev) && dev->bus && dev->bus->dev_name) - dev_set_name(dev, "%s%u", dev->bus->dev_name, dev->id); - - if (!dev_name(dev)) { + else if (dev->bus && dev->bus->dev_name) + error = dev_set_name(dev, "%s%u", dev->bus->dev_name, dev->id); + else error = -EINVAL; + if (error) goto name_error; - } pr_debug("device: '%s': %s\n", dev_name(dev), __func__); parent = get_device(dev->parent); kobj = get_device_parent(dev, parent); + if (IS_ERR(kobj)) { + error = PTR_ERR(kobj); + goto parent_error; + } if (kobj) dev->kobj.parent = kobj; /* use parent numa_node */ - if (parent) + if (parent && (dev_to_node(dev) == NUMA_NO_NODE)) set_dev_node(dev, dev_to_node(parent)); /* first, register with generic layer. */ /* we require the name to be set before, and pass NULL */ error = kobject_add(&dev->kobj, dev->kobj.parent, NULL); - if (error) + if (error) { + glue_dir = kobj; goto Error; + } /* notify platform of device entry */ - if (platform_notify) - platform_notify(dev); + device_platform_notify(dev); - error = device_create_file(dev, &uevent_attr); + error = device_create_file(dev, &dev_attr_uevent); if (error) goto attrError; - if (MAJOR(dev->devt)) { - error = device_create_file(dev, &devt_attr); - if (error) - goto ueventattrError; - - error = device_create_sys_dev_entry(dev); - if (error) - goto devtattrError; - - devtmpfs_create_node(dev); - } - error = device_add_class_symlinks(dev); if (error) goto SymlinkError; @@ -1123,63 +3651,101 @@ int device_add(struct device *dev) goto DPMError; device_pm_add(dev); + if (MAJOR(dev->devt)) { + error = device_create_file(dev, &dev_attr_dev); + if (error) + goto DevAttrError; + + error = device_create_sys_dev_entry(dev); + if (error) + goto SysEntryError; + + devtmpfs_create_node(dev); + } + /* Notify clients of device addition. This call must come * after dpm_sysfs_add() and before kobject_uevent(). */ - if (dev->bus) - blocking_notifier_call_chain(&dev->bus->p->bus_notifier, - BUS_NOTIFY_ADD_DEVICE, dev); - + bus_notify(dev, BUS_NOTIFY_ADD_DEVICE); kobject_uevent(&dev->kobj, KOBJ_ADD); + + /* + * Check if any of the other devices (consumers) have been waiting for + * this device (supplier) to be added so that they can create a device + * link to it. + * + * This needs to happen after device_pm_add() because device_link_add() + * requires the supplier be registered before it's called. + * + * But this also needs to happen before bus_probe_device() to make sure + * waiting consumers can link to it before the driver is bound to the + * device and the driver sync_state callback is called for this device. + */ + if (dev->fwnode && !dev->fwnode->dev) { + dev->fwnode->dev = dev; + fw_devlink_link_device(dev); + } + bus_probe_device(dev); + + /* + * If all driver registration is done and a newly added device doesn't + * match with any driver, don't block its consumers from probing in + * case the consumer device is able to operate without this supplier. + */ + if (dev->fwnode && fw_devlink_drv_reg_done && !dev->can_match) + fw_devlink_unblock_consumers(dev); + if (parent) klist_add_tail(&dev->p->knode_parent, &parent->p->klist_children); - if (dev->class) { - mutex_lock(&dev->class->p->mutex); + sp = class_to_subsys(dev->class); + if (sp) { + mutex_lock(&sp->mutex); /* tie the class to the device */ - klist_add_tail(&dev->knode_class, - &dev->class->p->klist_devices); + klist_add_tail(&dev->p->knode_class, &sp->klist_devices); /* notify any interfaces that the device is here */ - list_for_each_entry(class_intf, - &dev->class->p->interfaces, node) + list_for_each_entry(class_intf, &sp->interfaces, node) if (class_intf->add_dev) - class_intf->add_dev(dev, class_intf); - mutex_unlock(&dev->class->p->mutex); + class_intf->add_dev(dev); + mutex_unlock(&sp->mutex); + subsys_put(sp); } done: put_device(dev); return error; + SysEntryError: + if (MAJOR(dev->devt)) + device_remove_file(dev, &dev_attr_dev); + DevAttrError: + device_pm_remove(dev); + dpm_sysfs_remove(dev); DPMError: + device_set_driver(dev, NULL); bus_remove_device(dev); BusError: device_remove_attrs(dev); AttrsError: device_remove_class_symlinks(dev); SymlinkError: - if (MAJOR(dev->devt)) - devtmpfs_delete_node(dev); - if (MAJOR(dev->devt)) - device_remove_sys_dev_entry(dev); - devtattrError: - if (MAJOR(dev->devt)) - device_remove_file(dev, &devt_attr); - ueventattrError: - device_remove_file(dev, &uevent_attr); + device_remove_file(dev, &dev_attr_uevent); attrError: + device_platform_notify_remove(dev); kobject_uevent(&dev->kobj, KOBJ_REMOVE); + glue_dir = get_glue_dir(dev); kobject_del(&dev->kobj); Error: - cleanup_device_parent(dev); - if (parent) - put_device(parent); + cleanup_glue_dir(dev, glue_dir); +parent_error: + put_device(parent); name_error: kfree(dev->p); dev->p = NULL; goto done; } +EXPORT_SYMBOL_GPL(device_add); /** * device_register - register a device with the system. @@ -1204,6 +3770,7 @@ int device_register(struct device *dev) device_initialize(dev); return device_add(dev); } +EXPORT_SYMBOL_GPL(device_register); /** * get_device - increment reference count for device. @@ -1217,6 +3784,7 @@ struct device *get_device(struct device *dev) { return dev ? kobj_to_dev(kobject_get(&dev->kobj)) : NULL; } +EXPORT_SYMBOL_GPL(get_device); /** * put_device - decrement reference count. @@ -1228,6 +3796,25 @@ void put_device(struct device *dev) if (dev) kobject_put(&dev->kobj); } +EXPORT_SYMBOL_GPL(put_device); + +bool kill_device(struct device *dev) +{ + /* + * Require the device lock and set the "dead" flag to guarantee that + * the update behavior is consistent with the other bitfields near + * it and that we cannot have an asynchronous probe routine trying + * to run while we are tearing out the bus/class/sysfs from + * underneath the device. + */ + device_lock_assert(dev); + + if (dev->p->dead) + return false; + dev->p->dead = true; + return true; +} +EXPORT_SYMBOL_GPL(kill_device); /** * device_del - delete device from system. @@ -1244,52 +3831,76 @@ void put_device(struct device *dev) */ void device_del(struct device *dev) { + struct subsys_private *sp; struct device *parent = dev->parent; + struct kobject *glue_dir = NULL; struct class_interface *class_intf; + unsigned int noio_flag; + + device_lock(dev); + kill_device(dev); + device_unlock(dev); + + if (dev->fwnode && dev->fwnode->dev == dev) + dev->fwnode->dev = NULL; /* Notify clients of device removal. This call must come * before dpm_sysfs_remove(). */ - if (dev->bus) - blocking_notifier_call_chain(&dev->bus->p->bus_notifier, - BUS_NOTIFY_DEL_DEVICE, dev); + noio_flag = memalloc_noio_save(); + bus_notify(dev, BUS_NOTIFY_DEL_DEVICE); + dpm_sysfs_remove(dev); if (parent) klist_del(&dev->p->knode_parent); if (MAJOR(dev->devt)) { devtmpfs_delete_node(dev); device_remove_sys_dev_entry(dev); - device_remove_file(dev, &devt_attr); + device_remove_file(dev, &dev_attr_dev); } - if (dev->class) { + + sp = class_to_subsys(dev->class); + if (sp) { device_remove_class_symlinks(dev); - mutex_lock(&dev->class->p->mutex); + mutex_lock(&sp->mutex); /* notify any interfaces that the device is now gone */ - list_for_each_entry(class_intf, - &dev->class->p->interfaces, node) + list_for_each_entry(class_intf, &sp->interfaces, node) if (class_intf->remove_dev) - class_intf->remove_dev(dev, class_intf); + class_intf->remove_dev(dev); /* remove the device from the class list */ - klist_del(&dev->knode_class); - mutex_unlock(&dev->class->p->mutex); + klist_del(&dev->p->knode_class); + mutex_unlock(&sp->mutex); + subsys_put(sp); } - device_remove_file(dev, &uevent_attr); + device_remove_file(dev, &dev_attr_uevent); device_remove_attrs(dev); bus_remove_device(dev); device_pm_remove(dev); driver_deferred_probe_del(dev); + device_platform_notify_remove(dev); + device_links_purge(dev); - /* Notify the platform of the removal, in case they - * need to do anything... + /* + * If a device does not have a driver attached, we need to clean + * up any managed resources. We do this in device_release(), but + * it's never called (and we leak the device) if a managed + * resource holds a reference to the device. So release all + * managed resources here, like we do in driver_detach(). We + * still need to do so again in device_release() in case someone + * adds a new resource after this point, though. */ - if (platform_notify_remove) - platform_notify_remove(dev); + devres_release_all(dev); + + bus_notify(dev, BUS_NOTIFY_REMOVED_DEVICE); kobject_uevent(&dev->kobj, KOBJ_REMOVE); - cleanup_device_parent(dev); + glue_dir = get_glue_dir(dev); kobject_del(&dev->kobj); + cleanup_glue_dir(dev, glue_dir); + memalloc_noio_restore(noio_flag); put_device(parent); } +EXPORT_SYMBOL_GPL(device_del); /** * device_unregister - unregister device from system. @@ -1308,6 +3919,20 @@ void device_unregister(struct device *dev) device_del(dev); put_device(dev); } +EXPORT_SYMBOL_GPL(device_unregister); + +static struct device *prev_device(struct klist_iter *i) +{ + struct klist_node *n = klist_prev(i); + struct device *dev = NULL; + struct device_private *p; + + if (n) { + p = to_device_private_parent(n); + dev = p->device; + } + return dev; +} static struct device *next_device(struct klist_iter *i) { @@ -1335,7 +3960,7 @@ static struct device *next_device(struct klist_iter *i) * a name. This memory is returned in tmp and needs to be * freed by the caller. */ -const char *device_get_devnode(struct device *dev, +const char *device_get_devnode(const struct device *dev, umode_t *mode, kuid_t *uid, kgid_t *gid, const char **tmp) { @@ -1360,19 +3985,17 @@ const char *device_get_devnode(struct device *dev, return dev_name(dev); /* replace '!' in the name with '/' */ - *tmp = kstrdup(dev_name(dev), GFP_KERNEL); - if (!*tmp) + s = kstrdup_and_replace(dev_name(dev), '!', '/', GFP_KERNEL); + if (!s) return NULL; - while ((s = strchr(*tmp, '!'))) - s[0] = '/'; - return *tmp; + return *tmp = s; } /** * device_for_each_child - device child iterator. * @parent: parent struct device. - * @fn: function to be called for each device. * @data: data for the callback. + * @fn: function to be called for each device. * * Iterate over @parent's child devices, and call @fn for each, * passing it @data. @@ -1381,27 +4004,93 @@ const char *device_get_devnode(struct device *dev, * other than 0, we break out and return that value. */ int device_for_each_child(struct device *parent, void *data, - int (*fn)(struct device *dev, void *data)) + device_iter_t fn) +{ + struct klist_iter i; + struct device *child; + int error = 0; + + if (!parent || !parent->p) + return 0; + + klist_iter_init(&parent->p->klist_children, &i); + while (!error && (child = next_device(&i))) + error = fn(child, data); + klist_iter_exit(&i); + return error; +} +EXPORT_SYMBOL_GPL(device_for_each_child); + +/** + * device_for_each_child_reverse - device child iterator in reversed order. + * @parent: parent struct device. + * @data: data for the callback. + * @fn: function to be called for each device. + * + * Iterate over @parent's child devices, and call @fn for each, + * passing it @data. + * + * We check the return of @fn each time. If it returns anything + * other than 0, we break out and return that value. + */ +int device_for_each_child_reverse(struct device *parent, void *data, + device_iter_t fn) { struct klist_iter i; struct device *child; int error = 0; - if (!parent->p) + if (!parent || !parent->p) return 0; klist_iter_init(&parent->p->klist_children, &i); - while ((child = next_device(&i)) && !error) + while ((child = prev_device(&i)) && !error) error = fn(child, data); klist_iter_exit(&i); return error; } +EXPORT_SYMBOL_GPL(device_for_each_child_reverse); + +/** + * device_for_each_child_reverse_from - device child iterator in reversed order. + * @parent: parent struct device. + * @from: optional starting point in child list + * @data: data for the callback. + * @fn: function to be called for each device. + * + * Iterate over @parent's child devices, starting at @from, and call @fn + * for each, passing it @data. This helper is identical to + * device_for_each_child_reverse() when @from is NULL. + * + * @fn is checked each iteration. If it returns anything other than 0, + * iteration stop and that value is returned to the caller of + * device_for_each_child_reverse_from(); + */ +int device_for_each_child_reverse_from(struct device *parent, + struct device *from, void *data, + device_iter_t fn) +{ + struct klist_iter i; + struct device *child; + int error = 0; + + if (!parent || !parent->p) + return 0; + + klist_iter_init_node(&parent->p->klist_children, &i, + (from ? &from->p->knode_parent : NULL)); + while ((child = prev_device(&i)) && !error) + error = fn(child, data); + klist_iter_exit(&i); + return error; +} +EXPORT_SYMBOL_GPL(device_for_each_child_reverse_from); /** * device_find_child - device iterator for locating a particular device. * @parent: parent struct device - * @match: Callback function to check device * @data: Data to pass to match function + * @match: Callback function to check device * * This is similar to the device_for_each_child() function above, but it * returns a reference to a device that is 'found' for later use, as @@ -1414,22 +4103,26 @@ int device_for_each_child(struct device *parent, void *data, * * NOTE: you will need to drop the reference with put_device() after use. */ -struct device *device_find_child(struct device *parent, void *data, - int (*match)(struct device *dev, void *data)) +struct device *device_find_child(struct device *parent, const void *data, + device_match_t match) { struct klist_iter i; struct device *child; - if (!parent) + if (!parent || !parent->p) return NULL; klist_iter_init(&parent->p->klist_children, &i); - while ((child = next_device(&i))) - if (match(child, data) && get_device(child)) + while ((child = next_device(&i))) { + if (match(child, data)) { + get_device(child); break; + } + } klist_iter_exit(&i); return child; } +EXPORT_SYMBOL_GPL(device_find_child); int __init devices_init(void) { @@ -1445,9 +4138,14 @@ int __init devices_init(void) sysfs_dev_char_kobj = kobject_create_and_add("char", dev_kobj); if (!sysfs_dev_char_kobj) goto char_kobj_err; + device_link_wq = alloc_workqueue("device_link_wq", WQ_PERCPU, 0); + if (!device_link_wq) + goto wq_err; return 0; + wq_err: + kobject_put(sysfs_dev_char_kobj); char_kobj_err: kobject_put(sysfs_dev_block_kobj); block_kobj_err: @@ -1457,33 +4155,6 @@ int __init devices_init(void) return -ENOMEM; } -EXPORT_SYMBOL_GPL(device_for_each_child); -EXPORT_SYMBOL_GPL(device_find_child); - -EXPORT_SYMBOL_GPL(device_initialize); -EXPORT_SYMBOL_GPL(device_add); -EXPORT_SYMBOL_GPL(device_register); - -EXPORT_SYMBOL_GPL(device_del); -EXPORT_SYMBOL_GPL(device_unregister); -EXPORT_SYMBOL_GPL(get_device); -EXPORT_SYMBOL_GPL(put_device); - -EXPORT_SYMBOL_GPL(device_create_file); -EXPORT_SYMBOL_GPL(device_remove_file); - -static DEFINE_MUTEX(device_hotplug_lock); - -void lock_device_hotplug(void) -{ - mutex_lock(&device_hotplug_lock); -} - -void unlock_device_hotplug(void) -{ - mutex_unlock(&device_hotplug_lock); -} - static int device_check_offline(struct device *dev, void *not_used) { int ret; @@ -1667,39 +4338,16 @@ static void device_create_release(struct device *dev) kfree(dev); } -/** - * device_create_vargs - creates a device and registers it with sysfs - * @class: pointer to the struct class that this device should be registered to - * @parent: pointer to the parent struct device of this new device, if any - * @devt: the dev_t for the char device to be added - * @drvdata: the data to be added to the device for callbacks - * @fmt: string for the device's name - * @args: va_list for the device's name - * - * This function can be used by char device classes. A struct device - * will be created in sysfs, registered to the specified class. - * - * A "dev" file will be created, showing the dev_t for the device, if - * the dev_t is not 0,0. - * If a pointer to a parent struct device is passed in, the newly created - * struct device will be a child of that device in sysfs. - * The pointer to the struct device will be returned from the call. - * Any further sysfs files that might be required can be created using this - * pointer. - * - * Returns &struct device pointer on success, or ERR_PTR() on error. - * - * Note: the struct class passed to this function must have previously - * been created with a call to class_create(). - */ -struct device *device_create_vargs(struct class *class, struct device *parent, - dev_t devt, void *drvdata, const char *fmt, - va_list args) +static __printf(6, 0) struct device * +device_create_groups_vargs(const struct class *class, struct device *parent, + dev_t devt, void *drvdata, + const struct attribute_group **groups, + const char *fmt, va_list args) { struct device *dev = NULL; int retval = -ENODEV; - if (class == NULL || IS_ERR(class)) + if (IS_ERR_OR_NULL(class)) goto error; dev = kzalloc(sizeof(*dev), GFP_KERNEL); @@ -1708,9 +4356,11 @@ struct device *device_create_vargs(struct class *class, struct device *parent, goto error; } + device_initialize(dev); dev->devt = devt; dev->class = class; dev->parent = parent; + dev->groups = groups; dev->release = device_create_release; dev_set_drvdata(dev, drvdata); @@ -1718,7 +4368,7 @@ struct device *device_create_vargs(struct class *class, struct device *parent, if (retval) goto error; - retval = device_register(dev); + retval = device_add(dev); if (retval) goto error; @@ -1728,7 +4378,6 @@ error: put_device(dev); return ERR_PTR(retval); } -EXPORT_SYMBOL_GPL(device_create_vargs); /** * device_create - creates a device and registers it with sysfs @@ -1750,29 +4399,61 @@ EXPORT_SYMBOL_GPL(device_create_vargs); * pointer. * * Returns &struct device pointer on success, or ERR_PTR() on error. - * - * Note: the struct class passed to this function must have previously - * been created with a call to class_create(). */ -struct device *device_create(struct class *class, struct device *parent, +struct device *device_create(const struct class *class, struct device *parent, dev_t devt, void *drvdata, const char *fmt, ...) { va_list vargs; struct device *dev; va_start(vargs, fmt); - dev = device_create_vargs(class, parent, devt, drvdata, fmt, vargs); + dev = device_create_groups_vargs(class, parent, devt, drvdata, NULL, + fmt, vargs); va_end(vargs); return dev; } EXPORT_SYMBOL_GPL(device_create); -static int __match_devt(struct device *dev, const void *data) +/** + * device_create_with_groups - creates a device and registers it with sysfs + * @class: pointer to the struct class that this device should be registered to + * @parent: pointer to the parent struct device of this new device, if any + * @devt: the dev_t for the char device to be added + * @drvdata: the data to be added to the device for callbacks + * @groups: NULL-terminated list of attribute groups to be created + * @fmt: string for the device's name + * + * This function can be used by char device classes. A struct device + * will be created in sysfs, registered to the specified class. + * Additional attributes specified in the groups parameter will also + * be created automatically. + * + * A "dev" file will be created, showing the dev_t for the device, if + * the dev_t is not 0,0. + * If a pointer to a parent struct device is passed in, the newly created + * struct device will be a child of that device in sysfs. + * The pointer to the struct device will be returned from the call. + * Any further sysfs files that might be required can be created using this + * pointer. + * + * Returns &struct device pointer on success, or ERR_PTR() on error. + */ +struct device *device_create_with_groups(const struct class *class, + struct device *parent, dev_t devt, + void *drvdata, + const struct attribute_group **groups, + const char *fmt, ...) { - const dev_t *devt = data; + va_list vargs; + struct device *dev; - return dev->devt == *devt; + va_start(vargs, fmt); + dev = device_create_groups_vargs(class, parent, devt, drvdata, groups, + fmt, vargs); + va_end(vargs); + return dev; } +EXPORT_SYMBOL_GPL(device_create_with_groups); /** * device_destroy - removes a device that was created with device_create() @@ -1782,11 +4463,11 @@ static int __match_devt(struct device *dev, const void *data) * This call unregisters and cleans up a device that was created with a * call to device_create(). */ -void device_destroy(struct class *class, dev_t devt) +void device_destroy(const struct class *class, dev_t devt) { struct device *dev; - dev = class_find_device(class, NULL, &devt, __match_devt); + dev = class_find_device_by_devt(class, devt); if (dev) { put_device(dev); device_unregister(dev); @@ -1804,9 +4485,12 @@ EXPORT_SYMBOL_GPL(device_destroy); * on the same device to ensure that new_name is valid and * won't conflict with other devices. * - * Note: Don't call this function. Currently, the networking layer calls this - * function, but that will change. The following text from Kay Sievers offers - * some insight: + * Note: given that some subsystems (networking and infiniband) use this + * function, with no immediate plans for this to change, we cannot assume or + * require that this function not be called at all. + * + * However, if you're writing new code, do not call this function. The following + * text from Kay Sievers offers some insight: * * Renaming devices is racy at many levels, symlinks and other stuff are not * replaced atomically, and you get a "move" uevent, but it's not easy to @@ -1820,13 +4504,6 @@ EXPORT_SYMBOL_GPL(device_destroy); * kernel device renaming. Besides that, it's not even implemented now for * other things than (driver-core wise very simple) network devices. * - * We are currently about to change network renaming in udev to completely - * disallow renaming of devices in the same namespace as the kernel uses, - * because we can't solve the problems properly, that arise with swapping names - * of multiple interfaces without races. Means, renaming of eth[0-9]* will only - * be allowed to some other name than eth[0-9]*, for the aforementioned - * reasons. - * * Make up a "real" name in the driver before you register anything, or add * some other attributes for userspace to find the device, or use udev to add * symlinks -- but never rename kernel devices later, it's a complete mess. We @@ -1835,15 +4512,17 @@ EXPORT_SYMBOL_GPL(device_destroy); */ int device_rename(struct device *dev, const char *new_name) { + struct subsys_private *sp = NULL; + struct kobject *kobj = &dev->kobj; char *old_device_name = NULL; int error; + bool is_link_renamed = false; dev = get_device(dev); if (!dev) return -EINVAL; - pr_debug("device: '%s': %s: renaming to '%s'\n", dev_name(dev), - __func__, new_name); + dev_dbg(dev, "renaming to %s\n", new_name); old_device_name = kstrdup(dev_name(dev), GFP_KERNEL); if (!old_device_name) { @@ -1852,17 +4531,28 @@ int device_rename(struct device *dev, const char *new_name) } if (dev->class) { - error = sysfs_rename_link(&dev->class->p->subsys.kobj, - &dev->kobj, old_device_name, new_name); + sp = class_to_subsys(dev->class); + + if (!sp) { + error = -EINVAL; + goto out; + } + + error = sysfs_rename_link_ns(&sp->subsys.kobj, kobj, old_device_name, + new_name, kobject_namespace(kobj)); if (error) goto out; - } - error = kobject_rename(&dev->kobj, new_name); - if (error) - goto out; + is_link_renamed = true; + } + error = kobject_rename(kobj, new_name); out: + if (error && is_link_renamed) + sysfs_rename_link_ns(&sp->subsys.kobj, kobj, new_name, + old_device_name, kobject_namespace(kobj)); + subsys_put(sp); + put_device(dev); kfree(old_device_name); @@ -1888,7 +4578,7 @@ static int device_move_class_links(struct device *dev, /** * device_move - moves a device to a new parent * @dev: the pointer to the struct device to be moved - * @new_parent: the new parent of the device (can by NULL) + * @new_parent: the new parent of the device (can be NULL) * @dpm_order: how to reorder the dpm_list */ int device_move(struct device *dev, struct device *new_parent, @@ -1905,6 +4595,11 @@ int device_move(struct device *dev, struct device *new_parent, device_pm_lock(); new_parent = get_device(new_parent); new_parent_kobj = get_device_parent(dev, new_parent); + if (IS_ERR(new_parent_kobj)) { + error = PTR_ERR(new_parent_kobj); + put_device(new_parent); + goto out; + } pr_debug("device: '%s': %s: moving to '%s'\n", dev_name(dev), __func__, new_parent ? dev_name(new_parent) : "<NULL>"); @@ -1949,12 +4644,15 @@ int device_move(struct device *dev, struct device *new_parent, break; case DPM_ORDER_DEV_AFTER_PARENT: device_pm_move_after(dev, new_parent); + devices_kset_move_after(dev, new_parent); break; case DPM_ORDER_PARENT_BEFORE_DEV: device_pm_move_before(new_parent, dev); + devices_kset_move_before(new_parent, dev); break; case DPM_ORDER_DEV_LAST: device_pm_move_last(dev); + devices_kset_move_last(dev); break; } @@ -1966,12 +4664,136 @@ out: } EXPORT_SYMBOL_GPL(device_move); +static int device_attrs_change_owner(struct device *dev, kuid_t kuid, + kgid_t kgid) +{ + struct kobject *kobj = &dev->kobj; + const struct class *class = dev->class; + const struct device_type *type = dev->type; + int error; + + if (class) { + /* + * Change the device groups of the device class for @dev to + * @kuid/@kgid. + */ + error = sysfs_groups_change_owner(kobj, class->dev_groups, kuid, + kgid); + if (error) + return error; + } + + if (type) { + /* + * Change the device groups of the device type for @dev to + * @kuid/@kgid. + */ + error = sysfs_groups_change_owner(kobj, type->groups, kuid, + kgid); + if (error) + return error; + } + + /* Change the device groups of @dev to @kuid/@kgid. */ + error = sysfs_groups_change_owner(kobj, dev->groups, kuid, kgid); + if (error) + return error; + + if (device_supports_offline(dev) && !dev->offline_disabled) { + /* Change online device attributes of @dev to @kuid/@kgid. */ + error = sysfs_file_change_owner(kobj, dev_attr_online.attr.name, + kuid, kgid); + if (error) + return error; + } + + return 0; +} + +/** + * device_change_owner - change the owner of an existing device. + * @dev: device. + * @kuid: new owner's kuid + * @kgid: new owner's kgid + * + * This changes the owner of @dev and its corresponding sysfs entries to + * @kuid/@kgid. This function closely mirrors how @dev was added via driver + * core. + * + * Returns 0 on success or error code on failure. + */ +int device_change_owner(struct device *dev, kuid_t kuid, kgid_t kgid) +{ + int error; + struct kobject *kobj = &dev->kobj; + struct subsys_private *sp; + + dev = get_device(dev); + if (!dev) + return -EINVAL; + + /* + * Change the kobject and the default attributes and groups of the + * ktype associated with it to @kuid/@kgid. + */ + error = sysfs_change_owner(kobj, kuid, kgid); + if (error) + goto out; + + /* + * Change the uevent file for @dev to the new owner. The uevent file + * was created in a separate step when @dev got added and we mirror + * that step here. + */ + error = sysfs_file_change_owner(kobj, dev_attr_uevent.attr.name, kuid, + kgid); + if (error) + goto out; + + /* + * Change the device groups, the device groups associated with the + * device class, and the groups associated with the device type of @dev + * to @kuid/@kgid. + */ + error = device_attrs_change_owner(dev, kuid, kgid); + if (error) + goto out; + + error = dpm_sysfs_change_owner(dev, kuid, kgid); + if (error) + goto out; + + /* + * Change the owner of the symlink located in the class directory of + * the device class associated with @dev which points to the actual + * directory entry for @dev to @kuid/@kgid. This ensures that the + * symlink shows the same permissions as its target. + */ + sp = class_to_subsys(dev->class); + if (!sp) { + error = -EINVAL; + goto out; + } + error = sysfs_link_change_owner(&sp->subsys.kobj, &dev->kobj, dev_name(dev), kuid, kgid); + subsys_put(sp); + +out: + put_device(dev); + return error; +} +EXPORT_SYMBOL_GPL(device_change_owner); + /** * device_shutdown - call ->shutdown() on each device to shutdown. */ void device_shutdown(void) { - struct device *dev; + struct device *dev, *parent; + + wait_for_device_probe(); + device_block_probing(); + + cpufreq_suspend(); spin_lock(&devices_kset->list_lock); /* @@ -1988,7 +4810,7 @@ void device_shutdown(void) * prevent it from being freed because parent's * lock is to be held */ - get_device(dev->parent); + parent = get_device(dev->parent); get_device(dev); /* * Make sure the device is off the kset list, in the @@ -1998,14 +4820,19 @@ void device_shutdown(void) spin_unlock(&devices_kset->list_lock); /* hold lock to avoid race with probe/release */ - if (dev->parent) - device_lock(dev->parent); + if (parent) + device_lock(parent); device_lock(dev); /* Don't allow any more runtime suspends */ pm_runtime_get_noresume(dev); pm_runtime_barrier(dev); + if (dev->class && dev->class->shutdown_pre) { + if (initcall_debug) + dev_info(dev, "shutdown_pre\n"); + dev->class->shutdown_pre(dev); + } if (dev->bus && dev->bus->shutdown) { if (initcall_debug) dev_info(dev, "shutdown\n"); @@ -2017,16 +4844,15 @@ void device_shutdown(void) } device_unlock(dev); - if (dev->parent) - device_unlock(dev->parent); + if (parent) + device_unlock(parent); put_device(dev); - put_device(dev->parent); + put_device(parent); spin_lock(&devices_kset->list_lock); } spin_unlock(&devices_kset->list_lock); - async_synchronize_full(); } /* @@ -2034,20 +4860,21 @@ void device_shutdown(void) */ #ifdef CONFIG_PRINTK -static int -create_syslog_header(const struct device *dev, char *hdr, size_t hdrlen) +static void +set_dev_info(const struct device *dev, struct dev_printk_info *dev_info) { const char *subsys; - size_t pos = 0; + + memset(dev_info, 0, sizeof(*dev_info)); if (dev->class) subsys = dev->class->name; else if (dev->bus) subsys = dev->bus->name; else - return 0; + return; - pos += snprintf(hdr + pos, hdrlen - pos, "SUBSYSTEM=%s", subsys); + strscpy(dev_info->subsystem, subsys); /* * Add device identifier DEVICE=: @@ -2063,35 +4890,28 @@ create_syslog_header(const struct device *dev, char *hdr, size_t hdrlen) c = 'b'; else c = 'c'; - pos++; - pos += snprintf(hdr + pos, hdrlen - pos, - "DEVICE=%c%u:%u", - c, MAJOR(dev->devt), MINOR(dev->devt)); + + snprintf(dev_info->device, sizeof(dev_info->device), + "%c%u:%u", c, MAJOR(dev->devt), MINOR(dev->devt)); } else if (strcmp(subsys, "net") == 0) { struct net_device *net = to_net_dev(dev); - pos++; - pos += snprintf(hdr + pos, hdrlen - pos, - "DEVICE=n%u", net->ifindex); + snprintf(dev_info->device, sizeof(dev_info->device), + "n%u", net->ifindex); } else { - pos++; - pos += snprintf(hdr + pos, hdrlen - pos, - "DEVICE=+%s:%s", subsys, dev_name(dev)); + snprintf(dev_info->device, sizeof(dev_info->device), + "+%s:%s", subsys, dev_name(dev)); } - - return pos; } -EXPORT_SYMBOL(create_syslog_header); int dev_vprintk_emit(int level, const struct device *dev, const char *fmt, va_list args) { - char hdr[128]; - size_t hdrlen; + struct dev_printk_info dev_info; - hdrlen = create_syslog_header(dev, hdr, sizeof(hdr)); + set_dev_info(dev, &dev_info); - return vprintk_emit(0, level, hdrlen ? hdr : NULL, hdrlen, fmt, args); + return vprintk_emit(0, level, &dev_info, fmt, args); } EXPORT_SYMBOL(dev_vprintk_emit); @@ -2110,63 +4930,423 @@ int dev_printk_emit(int level, const struct device *dev, const char *fmt, ...) } EXPORT_SYMBOL(dev_printk_emit); -static int __dev_printk(const char *level, const struct device *dev, +static void __dev_printk(const char *level, const struct device *dev, struct va_format *vaf) { - if (!dev) - return printk("%s(NULL device *): %pV", level, vaf); - - return dev_printk_emit(level[1] - '0', dev, - "%s %s: %pV", - dev_driver_string(dev), dev_name(dev), vaf); + if (dev) + dev_printk_emit(level[1] - '0', dev, "%s %s: %pV", + dev_driver_string(dev), dev_name(dev), vaf); + else + printk("%s(NULL device *): %pV", level, vaf); } -int dev_printk(const char *level, const struct device *dev, - const char *fmt, ...) +void _dev_printk(const char *level, const struct device *dev, + const char *fmt, ...) { struct va_format vaf; va_list args; - int r; va_start(args, fmt); vaf.fmt = fmt; vaf.va = &args; - r = __dev_printk(level, dev, &vaf); + __dev_printk(level, dev, &vaf); va_end(args); - - return r; } -EXPORT_SYMBOL(dev_printk); +EXPORT_SYMBOL(_dev_printk); #define define_dev_printk_level(func, kern_level) \ -int func(const struct device *dev, const char *fmt, ...) \ +void func(const struct device *dev, const char *fmt, ...) \ { \ struct va_format vaf; \ va_list args; \ - int r; \ \ va_start(args, fmt); \ \ vaf.fmt = fmt; \ vaf.va = &args; \ \ - r = __dev_printk(kern_level, dev, &vaf); \ + __dev_printk(kern_level, dev, &vaf); \ \ va_end(args); \ - \ - return r; \ } \ EXPORT_SYMBOL(func); -define_dev_printk_level(dev_emerg, KERN_EMERG); -define_dev_printk_level(dev_alert, KERN_ALERT); -define_dev_printk_level(dev_crit, KERN_CRIT); -define_dev_printk_level(dev_err, KERN_ERR); -define_dev_printk_level(dev_warn, KERN_WARNING); -define_dev_printk_level(dev_notice, KERN_NOTICE); +define_dev_printk_level(_dev_emerg, KERN_EMERG); +define_dev_printk_level(_dev_alert, KERN_ALERT); +define_dev_printk_level(_dev_crit, KERN_CRIT); +define_dev_printk_level(_dev_err, KERN_ERR); +define_dev_printk_level(_dev_warn, KERN_WARNING); +define_dev_printk_level(_dev_notice, KERN_NOTICE); define_dev_printk_level(_dev_info, KERN_INFO); #endif + +static void __dev_probe_failed(const struct device *dev, int err, bool fatal, + const char *fmt, va_list vargsp) +{ + struct va_format vaf; + va_list vargs; + + /* + * On x86_64 and possibly on other architectures, va_list is actually a + * size-1 array containing a structure. As a result, function parameter + * vargsp decays from T[1] to T*, and &vargsp has type T** rather than + * T(*)[1], which is expected by its assignment to vaf.va below. + * + * One standard way to solve this mess is by creating a copy in a local + * variable of type va_list and then using a pointer to that local copy + * instead, which is the approach employed here. + */ + va_copy(vargs, vargsp); + + vaf.fmt = fmt; + vaf.va = &vargs; + + switch (err) { + case -EPROBE_DEFER: + device_set_deferred_probe_reason(dev, &vaf); + dev_dbg(dev, "error %pe: %pV", ERR_PTR(err), &vaf); + break; + + case -ENOMEM: + /* Don't print anything on -ENOMEM, there's already enough output */ + break; + + default: + /* Log fatal final failures as errors, otherwise produce warnings */ + if (fatal) + dev_err(dev, "error %pe: %pV", ERR_PTR(err), &vaf); + else + dev_warn(dev, "error %pe: %pV", ERR_PTR(err), &vaf); + break; + } + + va_end(vargs); +} + +/** + * dev_err_probe - probe error check and log helper + * @dev: the pointer to the struct device + * @err: error value to test + * @fmt: printf-style format string + * @...: arguments as specified in the format string + * + * This helper implements common pattern present in probe functions for error + * checking: print debug or error message depending if the error value is + * -EPROBE_DEFER and propagate error upwards. + * In case of -EPROBE_DEFER it sets also defer probe reason, which can be + * checked later by reading devices_deferred debugfs attribute. + * It replaces the following code sequence:: + * + * if (err != -EPROBE_DEFER) + * dev_err(dev, ...); + * else + * dev_dbg(dev, ...); + * return err; + * + * with:: + * + * return dev_err_probe(dev, err, ...); + * + * Using this helper in your probe function is totally fine even if @err + * is known to never be -EPROBE_DEFER. + * The benefit compared to a normal dev_err() is the standardized format + * of the error code, which is emitted symbolically (i.e. you get "EAGAIN" + * instead of "-35"), and having the error code returned allows more + * compact error paths. + * + * Returns @err. + */ +int dev_err_probe(const struct device *dev, int err, const char *fmt, ...) +{ + va_list vargs; + + va_start(vargs, fmt); + + /* Use dev_err() for logging when err doesn't equal -EPROBE_DEFER */ + __dev_probe_failed(dev, err, true, fmt, vargs); + + va_end(vargs); + + return err; +} +EXPORT_SYMBOL_GPL(dev_err_probe); + +/** + * dev_warn_probe - probe error check and log helper + * @dev: the pointer to the struct device + * @err: error value to test + * @fmt: printf-style format string + * @...: arguments as specified in the format string + * + * This helper implements common pattern present in probe functions for error + * checking: print debug or warning message depending if the error value is + * -EPROBE_DEFER and propagate error upwards. + * In case of -EPROBE_DEFER it sets also defer probe reason, which can be + * checked later by reading devices_deferred debugfs attribute. + * It replaces the following code sequence:: + * + * if (err != -EPROBE_DEFER) + * dev_warn(dev, ...); + * else + * dev_dbg(dev, ...); + * return err; + * + * with:: + * + * return dev_warn_probe(dev, err, ...); + * + * Using this helper in your probe function is totally fine even if @err + * is known to never be -EPROBE_DEFER. + * The benefit compared to a normal dev_warn() is the standardized format + * of the error code, which is emitted symbolically (i.e. you get "EAGAIN" + * instead of "-35"), and having the error code returned allows more + * compact error paths. + * + * Returns @err. + */ +int dev_warn_probe(const struct device *dev, int err, const char *fmt, ...) +{ + va_list vargs; + + va_start(vargs, fmt); + + /* Use dev_warn() for logging when err doesn't equal -EPROBE_DEFER */ + __dev_probe_failed(dev, err, false, fmt, vargs); + + va_end(vargs); + + return err; +} +EXPORT_SYMBOL_GPL(dev_warn_probe); + +static inline bool fwnode_is_primary(struct fwnode_handle *fwnode) +{ + return fwnode && !IS_ERR(fwnode->secondary); +} + +/** + * set_primary_fwnode - Change the primary firmware node of a given device. + * @dev: Device to handle. + * @fwnode: New primary firmware node of the device. + * + * Set the device's firmware node pointer to @fwnode, but if a secondary + * firmware node of the device is present, preserve it. + * + * Valid fwnode cases are: + * - primary --> secondary --> -ENODEV + * - primary --> NULL + * - secondary --> -ENODEV + * - NULL + */ +void set_primary_fwnode(struct device *dev, struct fwnode_handle *fwnode) +{ + struct device *parent = dev->parent; + struct fwnode_handle *fn = dev->fwnode; + + if (fwnode) { + if (fwnode_is_primary(fn)) + fn = fn->secondary; + + if (fn) { + WARN_ON(fwnode->secondary); + fwnode->secondary = fn; + } + dev->fwnode = fwnode; + } else { + if (fwnode_is_primary(fn)) { + dev->fwnode = fn->secondary; + + /* Skip nullifying fn->secondary if the primary is shared */ + if (parent && fn == parent->fwnode) + return; + + /* Set fn->secondary = NULL, so fn remains the primary fwnode */ + fn->secondary = NULL; + } else { + dev->fwnode = NULL; + } + } +} +EXPORT_SYMBOL_GPL(set_primary_fwnode); + +/** + * set_secondary_fwnode - Change the secondary firmware node of a given device. + * @dev: Device to handle. + * @fwnode: New secondary firmware node of the device. + * + * If a primary firmware node of the device is present, set its secondary + * pointer to @fwnode. Otherwise, set the device's firmware node pointer to + * @fwnode. + */ +void set_secondary_fwnode(struct device *dev, struct fwnode_handle *fwnode) +{ + if (fwnode) + fwnode->secondary = ERR_PTR(-ENODEV); + + if (fwnode_is_primary(dev->fwnode)) + dev->fwnode->secondary = fwnode; + else + dev->fwnode = fwnode; +} +EXPORT_SYMBOL_GPL(set_secondary_fwnode); + +/** + * device_remove_of_node - Remove an of_node from a device + * @dev: device whose device tree node is being removed + */ +void device_remove_of_node(struct device *dev) +{ + dev = get_device(dev); + if (!dev) + return; + + if (!dev->of_node) + goto end; + + if (dev->fwnode == of_fwnode_handle(dev->of_node)) + dev->fwnode = NULL; + + of_node_put(dev->of_node); + dev->of_node = NULL; + +end: + put_device(dev); +} +EXPORT_SYMBOL_GPL(device_remove_of_node); + +/** + * device_add_of_node - Add an of_node to an existing device + * @dev: device whose device tree node is being added + * @of_node: of_node to add + * + * Return: 0 on success or error code on failure. + */ +int device_add_of_node(struct device *dev, struct device_node *of_node) +{ + int ret; + + if (!of_node) + return -EINVAL; + + dev = get_device(dev); + if (!dev) + return -EINVAL; + + if (dev->of_node) { + dev_err(dev, "Cannot replace node %pOF with %pOF\n", + dev->of_node, of_node); + ret = -EBUSY; + goto end; + } + + dev->of_node = of_node_get(of_node); + + if (!dev->fwnode) + dev->fwnode = of_fwnode_handle(of_node); + + ret = 0; +end: + put_device(dev); + return ret; +} +EXPORT_SYMBOL_GPL(device_add_of_node); + +/** + * device_set_of_node_from_dev - reuse device-tree node of another device + * @dev: device whose device-tree node is being set + * @dev2: device whose device-tree node is being reused + * + * Takes another reference to the new device-tree node after first dropping + * any reference held to the old node. + */ +void device_set_of_node_from_dev(struct device *dev, const struct device *dev2) +{ + of_node_put(dev->of_node); + dev->of_node = of_node_get(dev2->of_node); + dev->of_node_reused = true; +} +EXPORT_SYMBOL_GPL(device_set_of_node_from_dev); + +void device_set_node(struct device *dev, struct fwnode_handle *fwnode) +{ + dev->fwnode = fwnode; + dev->of_node = to_of_node(fwnode); +} +EXPORT_SYMBOL_GPL(device_set_node); + +/** + * get_dev_from_fwnode - Obtain a reference count of the struct device the + * struct fwnode_handle is associated with. + * @fwnode: The pointer to the struct fwnode_handle to obtain the struct device + * reference count of. + * + * This function obtains a reference count of the device the device pointer + * embedded in the struct fwnode_handle points to. + * + * Note that the struct device pointer embedded in struct fwnode_handle does + * *not* have a reference count of the struct device itself. + * + * Hence, it is a UAF (and thus a bug) to call this function if the caller can't + * guarantee that the last reference count of the corresponding struct device is + * not dropped concurrently. + * + * This is possible since struct fwnode_handle has its own reference count and + * hence can out-live the struct device it is associated with. + */ +struct device *get_dev_from_fwnode(struct fwnode_handle *fwnode) +{ + return get_device((fwnode)->dev); +} +EXPORT_SYMBOL_GPL(get_dev_from_fwnode); + +int device_match_name(struct device *dev, const void *name) +{ + return sysfs_streq(dev_name(dev), name); +} +EXPORT_SYMBOL_GPL(device_match_name); + +int device_match_type(struct device *dev, const void *type) +{ + return dev->type == type; +} +EXPORT_SYMBOL_GPL(device_match_type); + +int device_match_of_node(struct device *dev, const void *np) +{ + return np && dev->of_node == np; +} +EXPORT_SYMBOL_GPL(device_match_of_node); + +int device_match_fwnode(struct device *dev, const void *fwnode) +{ + return fwnode && dev_fwnode(dev) == fwnode; +} +EXPORT_SYMBOL_GPL(device_match_fwnode); + +int device_match_devt(struct device *dev, const void *pdevt) +{ + return dev->devt == *(dev_t *)pdevt; +} +EXPORT_SYMBOL_GPL(device_match_devt); + +int device_match_acpi_dev(struct device *dev, const void *adev) +{ + return adev && ACPI_COMPANION(dev) == adev; +} +EXPORT_SYMBOL(device_match_acpi_dev); + +int device_match_acpi_handle(struct device *dev, const void *handle) +{ + return handle && ACPI_HANDLE(dev) == handle; +} +EXPORT_SYMBOL(device_match_acpi_handle); + +int device_match_any(struct device *dev, const void *unused) +{ + return 1; +} +EXPORT_SYMBOL_GPL(device_match_any); diff --git a/drivers/base/cpu.c b/drivers/base/cpu.c index a16d20e389f0..c6c57b6f61c6 100644 --- a/drivers/base/cpu.c +++ b/drivers/base/cpu.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * CPU subsystem support */ @@ -14,12 +15,18 @@ #include <linux/slab.h> #include <linux/percpu.h> #include <linux/acpi.h> +#include <linux/of.h> +#include <linux/cpufeature.h> +#include <linux/tick.h> +#include <linux/pm_qos.h> +#include <linux/delay.h> +#include <linux/sched/isolation.h> #include "base.h" static DEFINE_PER_CPU(struct device *, cpu_sys_devices); -static int cpu_subsys_match(struct device *dev, struct device_driver *drv) +static int cpu_subsys_match(struct device *dev, const struct device_driver *drv) { /* ACPI style match is the only one that may succeed. */ if (acpi_driver_match_device(dev, drv)) @@ -38,17 +45,36 @@ static void change_cpu_under_node(struct cpu *cpu, cpu->node_id = to_nid; } -static int __ref cpu_subsys_online(struct device *dev) +static int cpu_subsys_online(struct device *dev) { struct cpu *cpu = container_of(dev, struct cpu, dev); int cpuid = dev->id; int from_nid, to_nid; int ret; - - cpu_hotplug_driver_lock(); + int retries = 0; from_nid = cpu_to_node(cpuid); - ret = cpu_up(cpuid); + if (from_nid == NUMA_NO_NODE) + return -ENODEV; + +retry: + ret = cpu_device_up(dev); + + /* + * If -EBUSY is returned, it is likely that hotplug is temporarily + * disabled when cpu_hotplug_disable() was called. This condition is + * transient. So we retry after waiting for an exponentially + * increasing delay up to a total of at least 620ms as some PCI + * device initialization can take quite a while. + */ + if (ret == -EBUSY) { + retries++; + if (retries > 5) + return ret; + msleep(10 * (1 << retries)); + goto retry; + } + /* * When hot adding memory to memoryless node and enabling a cpu * on the node, node number of the cpu may internally change. @@ -57,24 +83,19 @@ static int __ref cpu_subsys_online(struct device *dev) if (from_nid != to_nid) change_cpu_under_node(cpu, from_nid, to_nid); - cpu_hotplug_driver_unlock(); return ret; } static int cpu_subsys_offline(struct device *dev) { - int ret; - - cpu_hotplug_driver_lock(); - ret = cpu_down(dev->id); - cpu_hotplug_driver_unlock(); - return ret; + return cpu_device_down(dev); } void unregister_cpu(struct cpu *cpu) { int logical_cpu = cpu->dev.id; + set_cpu_enabled(logical_cpu, false); unregister_cpu_under_node(logical_cpu, cpu_to_node(logical_cpu)); device_unregister(&cpu->dev); @@ -88,7 +109,17 @@ static ssize_t cpu_probe_store(struct device *dev, const char *buf, size_t count) { - return arch_cpu_probe(buf, count); + ssize_t cnt; + int ret; + + ret = lock_device_hotplug_sysfs(); + if (ret) + return ret; + + cnt = arch_cpu_probe(buf, count); + + unlock_device_hotplug(); + return cnt; } static ssize_t cpu_release_store(struct device *dev, @@ -96,7 +127,17 @@ static ssize_t cpu_release_store(struct device *dev, const char *buf, size_t count) { - return arch_cpu_release(buf, count); + ssize_t cnt; + int ret; + + ret = lock_device_hotplug_sysfs(); + if (ret) + return ret; + + cnt = arch_cpu_release(buf, count); + + unlock_device_hotplug(); + return cnt; } static DEVICE_ATTR(probe, S_IWUSR, NULL, cpu_probe_store); @@ -104,25 +145,14 @@ static DEVICE_ATTR(release, S_IWUSR, NULL, cpu_release_store); #endif /* CONFIG_ARCH_CPU_PROBE_RELEASE */ #endif /* CONFIG_HOTPLUG_CPU */ -struct bus_type cpu_subsys = { - .name = "cpu", - .dev_name = "cpu", - .match = cpu_subsys_match, -#ifdef CONFIG_HOTPLUG_CPU - .online = cpu_subsys_online, - .offline = cpu_subsys_offline, -#endif -}; -EXPORT_SYMBOL_GPL(cpu_subsys); - -#ifdef CONFIG_KEXEC +#ifdef CONFIG_CRASH_DUMP #include <linux/kexec.h> -static ssize_t show_crash_notes(struct device *dev, struct device_attribute *attr, +static ssize_t crash_notes_show(struct device *dev, + struct device_attribute *attr, char *buf) { struct cpu *cpu = container_of(dev, struct cpu, dev); - ssize_t rc; unsigned long long addr; int cpunum; @@ -135,21 +165,18 @@ static ssize_t show_crash_notes(struct device *dev, struct device_attribute *att * operation should be safe. No locking required. */ addr = per_cpu_ptr_to_phys(per_cpu_ptr(crash_notes, cpunum)); - rc = sprintf(buf, "%Lx\n", addr); - return rc; + + return sysfs_emit(buf, "%llx\n", addr); } -static DEVICE_ATTR(crash_notes, 0400, show_crash_notes, NULL); +static DEVICE_ATTR_ADMIN_RO(crash_notes); -static ssize_t show_crash_notes_size(struct device *dev, +static ssize_t crash_notes_size_show(struct device *dev, struct device_attribute *attr, char *buf) { - ssize_t rc; - - rc = sprintf(buf, "%zu\n", sizeof(note_buf_t)); - return rc; + return sysfs_emit(buf, "%zu\n", sizeof(note_buf_t)); } -static DEVICE_ATTR(crash_notes_size, 0400, show_crash_notes_size, NULL); +static DEVICE_ATTR_ADMIN_RO(crash_notes_size); static struct attribute *crash_note_cpu_attrs[] = { &dev_attr_crash_notes.attr, @@ -157,20 +184,20 @@ static struct attribute *crash_note_cpu_attrs[] = { NULL }; -static struct attribute_group crash_note_cpu_attr_group = { +static const struct attribute_group crash_note_cpu_attr_group = { .attrs = crash_note_cpu_attrs, }; #endif static const struct attribute_group *common_cpu_attr_groups[] = { -#ifdef CONFIG_KEXEC +#ifdef CONFIG_CRASH_DUMP &crash_note_cpu_attr_group, #endif NULL }; static const struct attribute_group *hotplugable_cpu_attr_groups[] = { -#ifdef CONFIG_KEXEC +#ifdef CONFIG_CRASH_DUMP &crash_note_cpu_attr_group, #endif NULL @@ -182,7 +209,7 @@ static const struct attribute_group *hotplugable_cpu_attr_groups[] = { struct cpu_attr { struct device_attribute attr; - const struct cpumask *const * const map; + const struct cpumask *const map; }; static ssize_t show_cpus_attr(struct device *dev, @@ -190,11 +217,8 @@ static ssize_t show_cpus_attr(struct device *dev, char *buf) { struct cpu_attr *ca = container_of(attr, struct cpu_attr, attr); - int n = cpulist_scnprintf(buf, PAGE_SIZE-2, *(ca->map)); - buf[n++] = '\n'; - buf[n] = '\0'; - return n; + return cpumap_print_to_pagebuf(true, buf, ca->map); } #define _CPU_ATTR(name, map) \ @@ -202,9 +226,9 @@ static ssize_t show_cpus_attr(struct device *dev, /* Keep in sync with cpu_subsys_attrs */ static struct cpu_attr cpu_attrs[] = { - _CPU_ATTR(online, &cpu_online_mask), - _CPU_ATTR(possible, &cpu_possible_mask), - _CPU_ATTR(present, &cpu_present_mask), + _CPU_ATTR(online, &__cpu_online_mask), + _CPU_ATTR(possible, &__cpu_possible_mask), + _CPU_ATTR(present, &__cpu_present_mask), }; /* @@ -213,8 +237,7 @@ static struct cpu_attr cpu_attrs[] = { static ssize_t print_cpus_kernel_max(struct device *dev, struct device_attribute *attr, char *buf) { - int n = snprintf(buf, PAGE_SIZE-2, "%d\n", NR_CPUS - 1); - return n; + return sysfs_emit(buf, "%d\n", NR_CPUS - 1); } static DEVICE_ATTR(kernel_max, 0444, print_cpus_kernel_max, NULL); @@ -224,40 +247,102 @@ unsigned int total_cpus; static ssize_t print_cpus_offline(struct device *dev, struct device_attribute *attr, char *buf) { - int n = 0, len = PAGE_SIZE-2; + int len = 0; cpumask_var_t offline; /* display offline cpus < nr_cpu_ids */ if (!alloc_cpumask_var(&offline, GFP_KERNEL)) return -ENOMEM; cpumask_andnot(offline, cpu_possible_mask, cpu_online_mask); - n = cpulist_scnprintf(buf, len, offline); + len += sysfs_emit_at(buf, len, "%*pbl", cpumask_pr_args(offline)); free_cpumask_var(offline); /* display offline cpus >= nr_cpu_ids */ if (total_cpus && nr_cpu_ids < total_cpus) { - if (n && n < len) - buf[n++] = ','; + len += sysfs_emit_at(buf, len, ","); if (nr_cpu_ids == total_cpus-1) - n += snprintf(&buf[n], len - n, "%d", nr_cpu_ids); + len += sysfs_emit_at(buf, len, "%u", nr_cpu_ids); else - n += snprintf(&buf[n], len - n, "%d-%d", - nr_cpu_ids, total_cpus-1); + len += sysfs_emit_at(buf, len, "%u-%d", + nr_cpu_ids, total_cpus - 1); } - n += snprintf(&buf[n], len - n, "\n"); - return n; + len += sysfs_emit_at(buf, len, "\n"); + + return len; } static DEVICE_ATTR(offline, 0444, print_cpus_offline, NULL); +static ssize_t print_cpus_enabled(struct device *dev, + struct device_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(cpu_enabled_mask)); +} +static DEVICE_ATTR(enabled, 0444, print_cpus_enabled, NULL); + +static ssize_t print_cpus_isolated(struct device *dev, + struct device_attribute *attr, char *buf) +{ + int len; + cpumask_var_t isolated; + + if (!alloc_cpumask_var(&isolated, GFP_KERNEL)) + return -ENOMEM; + + cpumask_andnot(isolated, cpu_possible_mask, + housekeeping_cpumask(HK_TYPE_DOMAIN)); + len = sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(isolated)); + + free_cpumask_var(isolated); + + return len; +} +static DEVICE_ATTR(isolated, 0444, print_cpus_isolated, NULL); + +static ssize_t housekeeping_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + const struct cpumask *hk_mask; + + hk_mask = housekeeping_cpumask(HK_TYPE_KERNEL_NOISE); + + if (housekeeping_enabled(HK_TYPE_KERNEL_NOISE)) + return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(hk_mask)); + return sysfs_emit(buf, "\n"); +} +static DEVICE_ATTR_RO(housekeeping); + +#ifdef CONFIG_NO_HZ_FULL +static ssize_t nohz_full_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + if (cpumask_available(tick_nohz_full_mask)) + return sysfs_emit(buf, "%*pbl\n", + cpumask_pr_args(tick_nohz_full_mask)); + return sysfs_emit(buf, "\n"); +} +static DEVICE_ATTR_RO(nohz_full); +#endif + +#ifdef CONFIG_CRASH_HOTPLUG +static ssize_t crash_hotplug_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + return sysfs_emit(buf, "%d\n", crash_check_hotplug_support()); +} +static DEVICE_ATTR_RO(crash_hotplug); +#endif + static void cpu_device_release(struct device *dev) { /* * This is an empty function to prevent the driver core from spitting a * warning at us. Yes, I know this is directly opposite of what the * documentation for the driver core and kobjects say, and the author - * of this code has already been publically ridiculed for doing + * of this code has already been publicly ridiculed for doing * something as foolish as this. However, at this point in time, it is * the only way to handle the issue of statically allocated cpu * devices. The different architectures will have their cpu device @@ -270,6 +355,56 @@ static void cpu_device_release(struct device *dev) */ } +#ifdef CONFIG_GENERIC_CPU_AUTOPROBE +static ssize_t print_cpu_modalias(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + int len = 0; + u32 i; + + len += sysfs_emit_at(buf, len, + "cpu:type:" CPU_FEATURE_TYPEFMT ":feature:", + CPU_FEATURE_TYPEVAL); + + for (i = 0; i < MAX_CPU_FEATURES; i++) + if (cpu_have_feature(i)) { + if (len + sizeof(",XXXX\n") >= PAGE_SIZE) { + WARN(1, "CPU features overflow page\n"); + break; + } + len += sysfs_emit_at(buf, len, ",%04X", i); + } + len += sysfs_emit_at(buf, len, "\n"); + return len; +} + +static int cpu_uevent(const struct device *dev, struct kobj_uevent_env *env) +{ + char *buf = kzalloc(PAGE_SIZE, GFP_KERNEL); + if (buf) { + print_cpu_modalias(NULL, NULL, buf); + add_uevent_var(env, "MODALIAS=%s", buf); + kfree(buf); + } + return 0; +} +#endif + +const struct bus_type cpu_subsys = { + .name = "cpu", + .dev_name = "cpu", + .match = cpu_subsys_match, +#ifdef CONFIG_HOTPLUG_CPU + .online = cpu_subsys_online, + .offline = cpu_subsys_offline, +#endif +#ifdef CONFIG_GENERIC_CPU_AUTOPROBE + .uevent = cpu_uevent, +#endif +}; +EXPORT_SYMBOL_GPL(cpu_subsys); + /* * register_cpu - Setup a sysfs device for a CPU. * @cpu - cpu->hotpluggable field set to 1 will generate a control file in @@ -278,7 +413,7 @@ static void cpu_device_release(struct device *dev) * * Initialize and register the CPU device. */ -int __cpuinit register_cpu(struct cpu *cpu, int num) +int register_cpu(struct cpu *cpu, int num) { int error; @@ -289,22 +424,26 @@ int __cpuinit register_cpu(struct cpu *cpu, int num) cpu->dev.release = cpu_device_release; cpu->dev.offline_disabled = !cpu->hotpluggable; cpu->dev.offline = !cpu_online(num); -#ifdef CONFIG_ARCH_HAS_CPU_AUTOPROBE - cpu->dev.bus->uevent = arch_cpu_uevent; -#endif + cpu->dev.of_node = of_get_cpu_node(num, NULL); cpu->dev.groups = common_cpu_attr_groups; if (cpu->hotpluggable) cpu->dev.groups = hotplugable_cpu_attr_groups; error = device_register(&cpu->dev); - if (!error) - per_cpu(cpu_sys_devices, num) = &cpu->dev; - if (!error) - register_cpu_under_node(num, cpu_to_node(num)); + if (error) { + put_device(&cpu->dev); + return error; + } + + per_cpu(cpu_sys_devices, num) = &cpu->dev; + register_cpu_under_node(num, cpu_to_node(num)); + dev_pm_qos_expose_latency_limit(&cpu->dev, + PM_QOS_RESUME_LATENCY_NO_CONSTRAINT); + set_cpu_enabled(num, true); - return error; + return 0; } -struct device *get_cpu_device(unsigned cpu) +struct device *get_cpu_device(unsigned int cpu) { if (cpu < nr_cpu_ids && cpu_possible(cpu)) return per_cpu(cpu_sys_devices, cpu); @@ -313,8 +452,62 @@ struct device *get_cpu_device(unsigned cpu) } EXPORT_SYMBOL_GPL(get_cpu_device); -#ifdef CONFIG_ARCH_HAS_CPU_AUTOPROBE -static DEVICE_ATTR(modalias, 0444, arch_print_cpu_modalias, NULL); +static void device_create_release(struct device *dev) +{ + kfree(dev); +} + +__printf(4, 0) +static struct device * +__cpu_device_create(struct device *parent, void *drvdata, + const struct attribute_group **groups, + const char *fmt, va_list args) +{ + struct device *dev = NULL; + int retval = -ENOMEM; + + dev = kzalloc(sizeof(*dev), GFP_KERNEL); + if (!dev) + goto error; + + device_initialize(dev); + dev->parent = parent; + dev->groups = groups; + dev->release = device_create_release; + device_set_pm_not_required(dev); + dev_set_drvdata(dev, drvdata); + + retval = kobject_set_name_vargs(&dev->kobj, fmt, args); + if (retval) + goto error; + + retval = device_add(dev); + if (retval) + goto error; + + return dev; + +error: + put_device(dev); + return ERR_PTR(retval); +} + +struct device *cpu_device_create(struct device *parent, void *drvdata, + const struct attribute_group **groups, + const char *fmt, ...) +{ + va_list vargs; + struct device *dev; + + va_start(vargs, fmt); + dev = __cpu_device_create(parent, drvdata, groups, fmt, vargs); + va_end(vargs); + return dev; +} +EXPORT_SYMBOL_GPL(cpu_device_create); + +#ifdef CONFIG_GENERIC_CPU_AUTOPROBE +static DEVICE_ATTR(modalias, 0444, print_cpu_modalias, NULL); #endif static struct attribute *cpu_root_attrs[] = { @@ -327,13 +520,22 @@ static struct attribute *cpu_root_attrs[] = { &cpu_attrs[2].attr.attr, &dev_attr_kernel_max.attr, &dev_attr_offline.attr, -#ifdef CONFIG_ARCH_HAS_CPU_AUTOPROBE + &dev_attr_enabled.attr, + &dev_attr_isolated.attr, + &dev_attr_housekeeping.attr, +#ifdef CONFIG_NO_HZ_FULL + &dev_attr_nohz_full.attr, +#endif +#ifdef CONFIG_CRASH_HOTPLUG + &dev_attr_crash_hotplug.attr, +#endif +#ifdef CONFIG_GENERIC_CPU_AUTOPROBE &dev_attr_modalias.attr, #endif NULL }; -static struct attribute_group cpu_root_attr_group = { +static const struct attribute_group cpu_root_attr_group = { .attrs = cpu_root_attrs, }; @@ -342,33 +544,153 @@ static const struct attribute_group *cpu_root_attr_groups[] = { NULL, }; -bool cpu_is_hotpluggable(unsigned cpu) +bool cpu_is_hotpluggable(unsigned int cpu) { struct device *dev = get_cpu_device(cpu); - return dev && container_of(dev, struct cpu, dev)->hotpluggable; + return dev && container_of(dev, struct cpu, dev)->hotpluggable + && tick_nohz_cpu_hotpluggable(cpu); } EXPORT_SYMBOL_GPL(cpu_is_hotpluggable); #ifdef CONFIG_GENERIC_CPU_DEVICES -static DEFINE_PER_CPU(struct cpu, cpu_devices); -#endif +DEFINE_PER_CPU(struct cpu, cpu_devices); + +bool __weak arch_cpu_is_hotpluggable(int cpu) +{ + return false; +} + +int __weak arch_register_cpu(int cpu) +{ + struct cpu *c = &per_cpu(cpu_devices, cpu); + + c->hotpluggable = arch_cpu_is_hotpluggable(cpu); + + return register_cpu(c, cpu); +} + +#ifdef CONFIG_HOTPLUG_CPU +void __weak arch_unregister_cpu(int num) +{ + unregister_cpu(&per_cpu(cpu_devices, num)); +} +#endif /* CONFIG_HOTPLUG_CPU */ +#endif /* CONFIG_GENERIC_CPU_DEVICES */ static void __init cpu_dev_register_generic(void) { -#ifdef CONFIG_GENERIC_CPU_DEVICES - int i; + int i, ret; + + if (!IS_ENABLED(CONFIG_GENERIC_CPU_DEVICES)) + return; - for_each_possible_cpu(i) { - if (register_cpu(&per_cpu(cpu_devices, i), i)) - panic("Failed to register CPU device"); + for_each_present_cpu(i) { + ret = arch_register_cpu(i); + if (ret && ret != -EPROBE_DEFER) + pr_warn("register_cpu %d failed (%d)\n", i, ret); + } +} + +#ifdef CONFIG_GENERIC_CPU_VULNERABILITIES +static ssize_t cpu_show_not_affected(struct device *dev, + struct device_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "Not affected\n"); +} + +#define CPU_SHOW_VULN_FALLBACK(func) \ + ssize_t cpu_show_##func(struct device *, \ + struct device_attribute *, char *) \ + __attribute__((weak, alias("cpu_show_not_affected"))) + +CPU_SHOW_VULN_FALLBACK(meltdown); +CPU_SHOW_VULN_FALLBACK(spectre_v1); +CPU_SHOW_VULN_FALLBACK(spectre_v2); +CPU_SHOW_VULN_FALLBACK(spec_store_bypass); +CPU_SHOW_VULN_FALLBACK(l1tf); +CPU_SHOW_VULN_FALLBACK(mds); +CPU_SHOW_VULN_FALLBACK(tsx_async_abort); +CPU_SHOW_VULN_FALLBACK(itlb_multihit); +CPU_SHOW_VULN_FALLBACK(srbds); +CPU_SHOW_VULN_FALLBACK(mmio_stale_data); +CPU_SHOW_VULN_FALLBACK(retbleed); +CPU_SHOW_VULN_FALLBACK(spec_rstack_overflow); +CPU_SHOW_VULN_FALLBACK(gds); +CPU_SHOW_VULN_FALLBACK(reg_file_data_sampling); +CPU_SHOW_VULN_FALLBACK(ghostwrite); +CPU_SHOW_VULN_FALLBACK(old_microcode); +CPU_SHOW_VULN_FALLBACK(indirect_target_selection); +CPU_SHOW_VULN_FALLBACK(tsa); +CPU_SHOW_VULN_FALLBACK(vmscape); + +static DEVICE_ATTR(meltdown, 0444, cpu_show_meltdown, NULL); +static DEVICE_ATTR(spectre_v1, 0444, cpu_show_spectre_v1, NULL); +static DEVICE_ATTR(spectre_v2, 0444, cpu_show_spectre_v2, NULL); +static DEVICE_ATTR(spec_store_bypass, 0444, cpu_show_spec_store_bypass, NULL); +static DEVICE_ATTR(l1tf, 0444, cpu_show_l1tf, NULL); +static DEVICE_ATTR(mds, 0444, cpu_show_mds, NULL); +static DEVICE_ATTR(tsx_async_abort, 0444, cpu_show_tsx_async_abort, NULL); +static DEVICE_ATTR(itlb_multihit, 0444, cpu_show_itlb_multihit, NULL); +static DEVICE_ATTR(srbds, 0444, cpu_show_srbds, NULL); +static DEVICE_ATTR(mmio_stale_data, 0444, cpu_show_mmio_stale_data, NULL); +static DEVICE_ATTR(retbleed, 0444, cpu_show_retbleed, NULL); +static DEVICE_ATTR(spec_rstack_overflow, 0444, cpu_show_spec_rstack_overflow, NULL); +static DEVICE_ATTR(gather_data_sampling, 0444, cpu_show_gds, NULL); +static DEVICE_ATTR(reg_file_data_sampling, 0444, cpu_show_reg_file_data_sampling, NULL); +static DEVICE_ATTR(ghostwrite, 0444, cpu_show_ghostwrite, NULL); +static DEVICE_ATTR(old_microcode, 0444, cpu_show_old_microcode, NULL); +static DEVICE_ATTR(indirect_target_selection, 0444, cpu_show_indirect_target_selection, NULL); +static DEVICE_ATTR(tsa, 0444, cpu_show_tsa, NULL); +static DEVICE_ATTR(vmscape, 0444, cpu_show_vmscape, NULL); + +static struct attribute *cpu_root_vulnerabilities_attrs[] = { + &dev_attr_meltdown.attr, + &dev_attr_spectre_v1.attr, + &dev_attr_spectre_v2.attr, + &dev_attr_spec_store_bypass.attr, + &dev_attr_l1tf.attr, + &dev_attr_mds.attr, + &dev_attr_tsx_async_abort.attr, + &dev_attr_itlb_multihit.attr, + &dev_attr_srbds.attr, + &dev_attr_mmio_stale_data.attr, + &dev_attr_retbleed.attr, + &dev_attr_spec_rstack_overflow.attr, + &dev_attr_gather_data_sampling.attr, + &dev_attr_reg_file_data_sampling.attr, + &dev_attr_ghostwrite.attr, + &dev_attr_old_microcode.attr, + &dev_attr_indirect_target_selection.attr, + &dev_attr_tsa.attr, + &dev_attr_vmscape.attr, + NULL +}; + +static const struct attribute_group cpu_root_vulnerabilities_group = { + .name = "vulnerabilities", + .attrs = cpu_root_vulnerabilities_attrs, +}; + +static void __init cpu_register_vulnerabilities(void) +{ + struct device *dev = bus_get_dev_root(&cpu_subsys); + + if (dev) { + if (sysfs_create_group(&dev->kobj, &cpu_root_vulnerabilities_group)) + pr_err("Unable to register CPU vulnerabilities\n"); + put_device(dev); } -#endif } +#else +static inline void cpu_register_vulnerabilities(void) { } +#endif + void __init cpu_dev_init(void) { if (subsys_system_register(&cpu_subsys, cpu_root_attr_groups)) panic("Failed to register CPU subsystem"); cpu_dev_register_generic(); + cpu_register_vulnerabilities(); } diff --git a/drivers/base/dd.c b/drivers/base/dd.c index 35fa36898916..349f31bedfa1 100644 --- a/drivers/base/dd.c +++ b/drivers/base/dd.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * drivers/base/dd.c - The core device/driver interactions. * @@ -13,18 +14,21 @@ * Copyright (c) 2002-3 Open Source Development Labs * Copyright (c) 2007-2009 Greg Kroah-Hartman <gregkh@suse.de> * Copyright (c) 2007-2009 Novell Inc. - * - * This file is released under the GPLv2 */ +#include <linux/debugfs.h> #include <linux/device.h> #include <linux/delay.h> +#include <linux/dma-map-ops.h> +#include <linux/init.h> #include <linux/module.h> #include <linux/kthread.h> #include <linux/wait.h> #include <linux/async.h> +#include <linux/pm_domain.h> #include <linux/pm_runtime.h> #include <linux/pinctrl/devinfo.h> +#include <linux/slab.h> #include "base.h" #include "power/power.h" @@ -51,9 +55,28 @@ static DEFINE_MUTEX(deferred_probe_mutex); static LIST_HEAD(deferred_probe_pending_list); static LIST_HEAD(deferred_probe_active_list); -static struct workqueue_struct *deferred_wq; +static atomic_t deferred_trigger_count = ATOMIC_INIT(0); +static bool initcalls_done; -/** +/* Save the async probe drivers' name from kernel cmdline */ +#define ASYNC_DRV_NAMES_MAX_LEN 256 +static char async_probe_drv_names[ASYNC_DRV_NAMES_MAX_LEN]; +static bool async_probe_default; + +/* + * In some cases, like suspend to RAM or hibernation, It might be reasonable + * to prohibit probing of devices as it could be unsafe. + * Once defer_all_probes is true all drivers probes will be forcibly deferred. + */ +static bool defer_all_probes; + +static void __device_set_deferred_probe_reason(const struct device *dev, char *reason) +{ + kfree(dev->p->deferred_probe_reason); + dev->p->deferred_probe_reason = reason; +} + +/* * deferred_probe_work_func() - Retry probing devices in the active list. */ static void deferred_probe_work_func(struct work_struct *work) @@ -81,6 +104,8 @@ static void deferred_probe_work_func(struct work_struct *work) get_device(dev); + __device_set_deferred_probe_reason(dev, NULL); + /* * Drop the mutex while probing each device; the probe path may * manipulate the deferred list @@ -93,13 +118,10 @@ static void deferred_probe_work_func(struct work_struct *work) * the list is a good order for suspend but deferred * probe makes that very unsafe. */ - device_pm_lock(); - device_pm_move_last(dev); - device_pm_unlock(); + device_pm_move_to_tail(dev); dev_dbg(dev, "Retrying from deferred list\n"); bus_probe_device(dev); - mutex_lock(&deferred_probe_mutex); put_device(dev); @@ -108,8 +130,11 @@ static void deferred_probe_work_func(struct work_struct *work) } static DECLARE_WORK(deferred_probe_work, deferred_probe_work_func); -static void driver_deferred_probe_add(struct device *dev) +void driver_deferred_probe_add(struct device *dev) { + if (!dev->can_match) + return; + mutex_lock(&deferred_probe_mutex); if (list_empty(&dev->p->deferred_probe)) { dev_dbg(dev, "Added to deferred list\n"); @@ -124,19 +149,31 @@ void driver_deferred_probe_del(struct device *dev) if (!list_empty(&dev->p->deferred_probe)) { dev_dbg(dev, "Removed from deferred list\n"); list_del_init(&dev->p->deferred_probe); + __device_set_deferred_probe_reason(dev, NULL); } mutex_unlock(&deferred_probe_mutex); } -static bool driver_deferred_probe_enable = false; +static bool driver_deferred_probe_enable; /** * driver_deferred_probe_trigger() - Kick off re-probing deferred devices * * This functions moves all devices from the pending list to the active * list and schedules the deferred probe workqueue to process them. It * should be called anytime a driver is successfully bound to a device. + * + * Note, there is a race condition in multi-threaded probe. In the case where + * more than one device is probing at the same time, it is possible for one + * probe to complete successfully while another is about to defer. If the second + * depends on the first, then it will get put on the pending list after the + * trigger event has already occurred and will be stuck there. + * + * The atomic 'deferred_trigger_count' is used to determine if a successful + * trigger has occurred in the midst of probing a driver. If the trigger count + * changes in the midst of a probe, then deferred processing should be triggered + * again. */ -static void driver_deferred_probe_trigger(void) +void driver_deferred_probe_trigger(void) { if (!driver_deferred_probe_enable) return; @@ -147,6 +184,7 @@ static void driver_deferred_probe_trigger(void) * into the active list so they can be retried by the workqueue */ mutex_lock(&deferred_probe_mutex); + atomic_inc(&deferred_trigger_count); list_splice_tail_init(&deferred_probe_pending_list, &deferred_probe_active_list); mutex_unlock(&deferred_probe_mutex); @@ -155,7 +193,146 @@ static void driver_deferred_probe_trigger(void) * Kick the re-probe thread. It may already be scheduled, but it is * safe to kick it again. */ - queue_work(deferred_wq, &deferred_probe_work); + queue_work(system_dfl_wq, &deferred_probe_work); +} + +/** + * device_block_probing() - Block/defer device's probes + * + * It will disable probing of devices and defer their probes instead. + */ +void device_block_probing(void) +{ + defer_all_probes = true; + /* sync with probes to avoid races. */ + wait_for_device_probe(); +} + +/** + * device_unblock_probing() - Unblock/enable device's probes + * + * It will restore normal behavior and trigger re-probing of deferred + * devices. + */ +void device_unblock_probing(void) +{ + defer_all_probes = false; + driver_deferred_probe_trigger(); +} + +/** + * device_set_deferred_probe_reason() - Set defer probe reason message for device + * @dev: the pointer to the struct device + * @vaf: the pointer to va_format structure with message + */ +void device_set_deferred_probe_reason(const struct device *dev, struct va_format *vaf) +{ + const char *drv = dev_driver_string(dev); + char *reason; + + mutex_lock(&deferred_probe_mutex); + + reason = kasprintf(GFP_KERNEL, "%s: %pV", drv, vaf); + __device_set_deferred_probe_reason(dev, reason); + + mutex_unlock(&deferred_probe_mutex); +} + +/* + * deferred_devs_show() - Show the devices in the deferred probe pending list. + */ +static int deferred_devs_show(struct seq_file *s, void *data) +{ + struct device_private *curr; + + mutex_lock(&deferred_probe_mutex); + + list_for_each_entry(curr, &deferred_probe_pending_list, deferred_probe) + seq_printf(s, "%s\t%s", dev_name(curr->device), + curr->deferred_probe_reason ?: "\n"); + + mutex_unlock(&deferred_probe_mutex); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(deferred_devs); + +#ifdef CONFIG_MODULES +static int driver_deferred_probe_timeout = 10; +#else +static int driver_deferred_probe_timeout; +#endif + +static int __init deferred_probe_timeout_setup(char *str) +{ + int timeout; + + if (!kstrtoint(str, 10, &timeout)) + driver_deferred_probe_timeout = timeout; + return 1; +} +__setup("deferred_probe_timeout=", deferred_probe_timeout_setup); + +/** + * driver_deferred_probe_check_state() - Check deferred probe state + * @dev: device to check + * + * Return: + * * -ENODEV if initcalls have completed and modules are disabled. + * * -ETIMEDOUT if the deferred probe timeout was set and has expired + * and modules are enabled. + * * -EPROBE_DEFER in other cases. + * + * Drivers or subsystems can opt-in to calling this function instead of directly + * returning -EPROBE_DEFER. + */ +int driver_deferred_probe_check_state(struct device *dev) +{ + if (!IS_ENABLED(CONFIG_MODULES) && initcalls_done) { + dev_warn(dev, "ignoring dependency for device, assuming no driver\n"); + return -ENODEV; + } + + if (!driver_deferred_probe_timeout && initcalls_done) { + dev_warn(dev, "deferred probe timeout, ignoring dependency\n"); + return -ETIMEDOUT; + } + + return -EPROBE_DEFER; +} +EXPORT_SYMBOL_GPL(driver_deferred_probe_check_state); + +static void deferred_probe_timeout_work_func(struct work_struct *work) +{ + struct device_private *p; + + fw_devlink_drivers_done(); + + driver_deferred_probe_timeout = 0; + driver_deferred_probe_trigger(); + flush_work(&deferred_probe_work); + + mutex_lock(&deferred_probe_mutex); + list_for_each_entry(p, &deferred_probe_pending_list, deferred_probe) + dev_warn(p->device, "deferred probe pending: %s", p->deferred_probe_reason ?: "(reason unknown)\n"); + mutex_unlock(&deferred_probe_mutex); + + fw_devlink_probing_done(); +} +static DECLARE_DELAYED_WORK(deferred_probe_timeout_work, deferred_probe_timeout_work_func); + +void deferred_probe_extend_timeout(void) +{ + /* + * If the work hasn't been queued yet or if the work expired, don't + * start a new one. + */ + if (cancel_delayed_work(&deferred_probe_timeout_work)) { + schedule_delayed_work(&deferred_probe_timeout_work, + driver_deferred_probe_timeout * HZ); + pr_debug("Extended deferred probe timeout by %d secs\n", + driver_deferred_probe_timeout); + } } /** @@ -167,30 +344,72 @@ static void driver_deferred_probe_trigger(void) */ static int deferred_probe_initcall(void) { - deferred_wq = create_singlethread_workqueue("deferwq"); - if (WARN_ON(!deferred_wq)) - return -ENOMEM; + debugfs_create_file("devices_deferred", 0444, NULL, NULL, + &deferred_devs_fops); driver_deferred_probe_enable = true; driver_deferred_probe_trigger(); /* Sort as many dependencies as possible before exiting initcalls */ - flush_workqueue(deferred_wq); + flush_work(&deferred_probe_work); + initcalls_done = true; + + if (!IS_ENABLED(CONFIG_MODULES)) + fw_devlink_drivers_done(); + + /* + * Trigger deferred probe again, this time we won't defer anything + * that is optional + */ + driver_deferred_probe_trigger(); + flush_work(&deferred_probe_work); + + if (driver_deferred_probe_timeout > 0) { + schedule_delayed_work(&deferred_probe_timeout_work, + driver_deferred_probe_timeout * HZ); + } + + if (!IS_ENABLED(CONFIG_MODULES)) + fw_devlink_probing_done(); + return 0; } late_initcall(deferred_probe_initcall); +static void __exit deferred_probe_exit(void) +{ + debugfs_lookup_and_remove("devices_deferred", NULL); +} +__exitcall(deferred_probe_exit); + +/** + * device_is_bound() - Check if device is bound to a driver + * @dev: device to check + * + * Returns true if passed device has already finished probing successfully + * against a driver. + * + * This function must be called with the device lock held. + */ +bool device_is_bound(struct device *dev) +{ + return dev->p && klist_node_attached(&dev->p->knode_driver); +} +EXPORT_SYMBOL_GPL(device_is_bound); + static void driver_bound(struct device *dev) { - if (klist_node_attached(&dev->p->knode_driver)) { - printk(KERN_WARNING "%s: device %s already bound\n", - __func__, kobject_name(&dev->kobj)); + if (device_is_bound(dev)) { + dev_warn(dev, "%s: device already bound\n", __func__); return; } - pr_debug("driver: '%s': %s: bound to device '%s'\n", dev_name(dev), - __func__, dev->driver->name); + dev_dbg(dev, "driver: '%s': %s: bound to device\n", dev->driver->name, + __func__); klist_add_tail(&dev->p->knode_driver, &dev->driver->p->klist_devices); + device_links_driver_bound(dev); + + device_pm_check_callbacks(dev); /* * Make sure the device is no longer in one of the deferred lists and @@ -199,28 +418,51 @@ static void driver_bound(struct device *dev) driver_deferred_probe_del(dev); driver_deferred_probe_trigger(); - if (dev->bus) - blocking_notifier_call_chain(&dev->bus->p->bus_notifier, - BUS_NOTIFY_BOUND_DRIVER, dev); + bus_notify(dev, BUS_NOTIFY_BOUND_DRIVER); + kobject_uevent(&dev->kobj, KOBJ_BIND); } +static ssize_t coredump_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + device_lock(dev); + dev->driver->coredump(dev); + device_unlock(dev); + + return count; +} +static DEVICE_ATTR_WO(coredump); + static int driver_sysfs_add(struct device *dev) { int ret; - if (dev->bus) - blocking_notifier_call_chain(&dev->bus->p->bus_notifier, - BUS_NOTIFY_BIND_DRIVER, dev); + bus_notify(dev, BUS_NOTIFY_BIND_DRIVER); ret = sysfs_create_link(&dev->driver->p->kobj, &dev->kobj, + kobject_name(&dev->kobj)); + if (ret) + goto fail; + + ret = sysfs_create_link(&dev->kobj, &dev->driver->p->kobj, + "driver"); + if (ret) + goto rm_dev; + + if (!IS_ENABLED(CONFIG_DEV_COREDUMP) || !dev->driver->coredump) + return 0; + + ret = device_create_file(dev, &dev_attr_coredump); + if (!ret) + return 0; + + sysfs_remove_link(&dev->kobj, "driver"); + +rm_dev: + sysfs_remove_link(&dev->driver->p->kobj, kobject_name(&dev->kobj)); - if (ret == 0) { - ret = sysfs_create_link(&dev->kobj, &dev->driver->p->kobj, - "driver"); - if (ret) - sysfs_remove_link(&dev->driver->p->kobj, - kobject_name(&dev->kobj)); - } + +fail: return ret; } @@ -229,6 +471,8 @@ static void driver_sysfs_remove(struct device *dev) struct device_driver *drv = dev->driver; if (drv) { + if (drv->coredump) + device_remove_file(dev, &dev_attr_coredump); sysfs_remove_link(&drv->p->kobj, kobject_name(&dev->kobj)); sysfs_remove_link(&dev->kobj, "driver"); } @@ -241,20 +485,25 @@ static void driver_sysfs_remove(struct device *dev) * Allow manual attachment of a driver to a device. * Caller must have already set @dev->driver. * - * Note that this does not modify the bus reference count - * nor take the bus's rwsem. Please verify those are accounted - * for before calling this. (It is ok to call with no other effort - * from a driver's probe() method.) + * Note that this does not modify the bus reference count. + * Please verify that is accounted for before calling this. + * (It is ok to call with no other effort from a driver's probe() method.) * * This function must be called with the device lock held. + * + * Callers should prefer to use device_driver_attach() instead. */ int device_bind_driver(struct device *dev) { int ret; ret = driver_sysfs_add(dev); - if (!ret) + if (!ret) { + device_links_force_bind(dev); driver_bound(dev); + } + else + bus_notify(dev, BUS_NOTIFY_DRIVER_NOT_BOUND); return ret; } EXPORT_SYMBOL_GPL(device_bind_driver); @@ -262,71 +511,240 @@ EXPORT_SYMBOL_GPL(device_bind_driver); static atomic_t probe_count = ATOMIC_INIT(0); static DECLARE_WAIT_QUEUE_HEAD(probe_waitqueue); -static int really_probe(struct device *dev, struct device_driver *drv) +static ssize_t state_synced_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) { int ret = 0; - atomic_inc(&probe_count); - pr_debug("bus: '%s': %s: probing driver %s with device %s\n", - drv->bus->name, __func__, drv->name, dev_name(dev)); - WARN_ON(!list_empty(&dev->devres_head)); + if (strcmp("1", buf)) + return -EINVAL; + + device_lock(dev); + if (!dev->state_synced) { + dev->state_synced = true; + dev_sync_state(dev); + } else { + ret = -EINVAL; + } + device_unlock(dev); + + return ret ? ret : count; +} + +static ssize_t state_synced_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + bool val; + + device_lock(dev); + val = dev->state_synced; + device_unlock(dev); + + return sysfs_emit(buf, "%u\n", val); +} +static DEVICE_ATTR_RW(state_synced); + +static void device_unbind_cleanup(struct device *dev) +{ + devres_release_all(dev); + arch_teardown_dma_ops(dev); + kfree(dev->dma_range_map); + dev->dma_range_map = NULL; + device_set_driver(dev, NULL); + dev_set_drvdata(dev, NULL); + dev_pm_domain_detach(dev, dev->power.detach_power_off); + if (dev->pm_domain && dev->pm_domain->dismiss) + dev->pm_domain->dismiss(dev); + pm_runtime_reinit(dev); + dev_pm_set_driver_flags(dev, 0); +} + +static void device_remove(struct device *dev) +{ + device_remove_file(dev, &dev_attr_state_synced); + device_remove_groups(dev, dev->driver->dev_groups); + + if (dev->bus && dev->bus->remove) + dev->bus->remove(dev); + else if (dev->driver->remove) + dev->driver->remove(dev); +} + +static int call_driver_probe(struct device *dev, const struct device_driver *drv) +{ + int ret = 0; + + if (dev->bus->probe) + ret = dev->bus->probe(dev); + else if (drv->probe) + ret = drv->probe(dev); + + switch (ret) { + case 0: + break; + case -EPROBE_DEFER: + /* Driver requested deferred probing */ + dev_dbg(dev, "Driver %s requests probe deferral\n", drv->name); + break; + case -ENODEV: + case -ENXIO: + dev_dbg(dev, "probe with driver %s rejects match %d\n", + drv->name, ret); + break; + default: + /* driver matched but the probe failed */ + dev_err(dev, "probe with driver %s failed with error %d\n", + drv->name, ret); + break; + } + + return ret; +} + +static int really_probe(struct device *dev, const struct device_driver *drv) +{ + bool test_remove = IS_ENABLED(CONFIG_DEBUG_TEST_DRIVER_REMOVE) && + !drv->suppress_bind_attrs; + int ret, link_ret; + + if (defer_all_probes) { + /* + * Value of defer_all_probes can be set only by + * device_block_probing() which, in turn, will call + * wait_for_device_probe() right after that to avoid any races. + */ + dev_dbg(dev, "Driver %s force probe deferral\n", drv->name); + return -EPROBE_DEFER; + } - dev->driver = drv; + link_ret = device_links_check_suppliers(dev); + if (link_ret == -EPROBE_DEFER) + return link_ret; + + dev_dbg(dev, "bus: '%s': %s: probing driver %s with device\n", + drv->bus->name, __func__, drv->name); + if (!list_empty(&dev->devres_head)) { + dev_crit(dev, "Resources present before probing\n"); + ret = -EBUSY; + goto done; + } + +re_probe: + device_set_driver(dev, drv); /* If using pinctrl, bind pins now before probing */ ret = pinctrl_bind_pins(dev); if (ret) - goto probe_failed; + goto pinctrl_bind_failed; - if (driver_sysfs_add(dev)) { - printk(KERN_ERR "%s: driver_sysfs_add(%s) failed\n", - __func__, dev_name(dev)); - goto probe_failed; + if (dev->bus->dma_configure) { + ret = dev->bus->dma_configure(dev); + if (ret) + goto pinctrl_bind_failed; } - if (dev->bus->probe) { - ret = dev->bus->probe(dev); - if (ret) - goto probe_failed; - } else if (drv->probe) { - ret = drv->probe(dev); + ret = driver_sysfs_add(dev); + if (ret) { + dev_err(dev, "%s: driver_sysfs_add failed\n", __func__); + goto sysfs_failed; + } + + if (dev->pm_domain && dev->pm_domain->activate) { + ret = dev->pm_domain->activate(dev); if (ret) goto probe_failed; } + ret = call_driver_probe(dev, drv); + if (ret) { + /* + * If fw_devlink_best_effort is active (denoted by -EAGAIN), the + * device might actually probe properly once some of its missing + * suppliers have probed. So, treat this as if the driver + * returned -EPROBE_DEFER. + */ + if (link_ret == -EAGAIN) + ret = -EPROBE_DEFER; + + /* + * Return probe errors as positive values so that the callers + * can distinguish them from other errors. + */ + ret = -ret; + goto probe_failed; + } + + ret = device_add_groups(dev, drv->dev_groups); + if (ret) { + dev_err(dev, "device_add_groups() failed\n"); + goto dev_groups_failed; + } + + if (dev_has_sync_state(dev)) { + ret = device_create_file(dev, &dev_attr_state_synced); + if (ret) { + dev_err(dev, "state_synced sysfs add failed\n"); + goto dev_sysfs_state_synced_failed; + } + } + + if (test_remove) { + test_remove = false; + + device_remove(dev); + driver_sysfs_remove(dev); + if (dev->bus && dev->bus->dma_cleanup) + dev->bus->dma_cleanup(dev); + device_unbind_cleanup(dev); + + goto re_probe; + } + + pinctrl_init_done(dev); + + if (dev->pm_domain && dev->pm_domain->sync) + dev->pm_domain->sync(dev); + driver_bound(dev); - ret = 1; - pr_debug("bus: '%s': %s: bound device %s to driver %s\n", - drv->bus->name, __func__, dev_name(dev), drv->name); + dev_dbg(dev, "bus: '%s': %s: bound device to driver %s\n", + drv->bus->name, __func__, drv->name); goto done; +dev_sysfs_state_synced_failed: +dev_groups_failed: + device_remove(dev); probe_failed: - devres_release_all(dev); driver_sysfs_remove(dev); - dev->driver = NULL; - dev_set_drvdata(dev, NULL); +sysfs_failed: + bus_notify(dev, BUS_NOTIFY_DRIVER_NOT_BOUND); + if (dev->bus && dev->bus->dma_cleanup) + dev->bus->dma_cleanup(dev); +pinctrl_bind_failed: + device_links_no_driver(dev); + device_unbind_cleanup(dev); +done: + return ret; +} - if (ret == -EPROBE_DEFER) { - /* Driver requested deferred probing */ - dev_info(dev, "Driver %s requests probe deferral\n", drv->name); - driver_deferred_probe_add(dev); - } else if (ret != -ENODEV && ret != -ENXIO) { - /* driver matched but the probe failed */ - printk(KERN_WARNING - "%s: probe of %s failed with error %d\n", - drv->name, dev_name(dev), ret); - } else { - pr_debug("%s: probe of %s rejects match %d\n", - drv->name, dev_name(dev), ret); - } +/* + * For initcall_debug, show the driver probe time. + */ +static int really_probe_debug(struct device *dev, const struct device_driver *drv) +{ + ktime_t calltime, rettime; + int ret; + + calltime = ktime_get(); + ret = really_probe(dev, drv); + rettime = ktime_get(); /* - * Ignore errors returned by ->probe so that the next driver can try - * its luck. + * Don't change this to pr_debug() because that requires + * CONFIG_DYNAMIC_DEBUG and we want a simple 'initcall_debug' on the + * kernel commandline to print this all the time at the debug level. */ - ret = 0; -done: - atomic_dec(&probe_count); - wake_up(&probe_waitqueue); + printk(KERN_DEBUG "probe of %s returned %d after %lld usecs\n", + dev_name(dev), ret, ktime_us_delta(rettime, calltime)); return ret; } @@ -336,13 +754,12 @@ done: * * Should somehow figure out how to use a semaphore, not an atomic variable... */ -int driver_probe_done(void) +bool __init driver_probe_done(void) { - pr_debug("%s: probe_count = %d\n", __func__, - atomic_read(&probe_count)); - if (atomic_read(&probe_count)) - return -EBUSY; - return 0; + int local_probe_count = atomic_read(&probe_count); + + pr_debug("%s: probe_count = %d\n", __func__, local_probe_count); + return !local_probe_count; } /** @@ -351,48 +768,291 @@ int driver_probe_done(void) */ void wait_for_device_probe(void) { + /* wait for the deferred probe workqueue to finish */ + flush_work(&deferred_probe_work); + /* wait for the known devices to complete their probing */ wait_event(probe_waitqueue, atomic_read(&probe_count) == 0); async_synchronize_full(); } EXPORT_SYMBOL_GPL(wait_for_device_probe); +static int __driver_probe_device(const struct device_driver *drv, struct device *dev) +{ + int ret = 0; + + if (dev->p->dead || !device_is_registered(dev)) + return -ENODEV; + if (dev->driver) + return -EBUSY; + + dev->can_match = true; + dev_dbg(dev, "bus: '%s': %s: matched device with driver %s\n", + drv->bus->name, __func__, drv->name); + + pm_runtime_get_suppliers(dev); + if (dev->parent) + pm_runtime_get_sync(dev->parent); + + pm_runtime_barrier(dev); + if (initcall_debug) + ret = really_probe_debug(dev, drv); + else + ret = really_probe(dev, drv); + pm_request_idle(dev); + + if (dev->parent) + pm_runtime_put(dev->parent); + + pm_runtime_put_suppliers(dev); + return ret; +} + /** * driver_probe_device - attempt to bind device & driver together * @drv: driver to bind a device to * @dev: device to try to bind to the driver * - * This function returns -ENODEV if the device is not registered, - * 1 if the device is bound successfully and 0 otherwise. + * This function returns -ENODEV if the device is not registered, -EBUSY if it + * already has a driver, 0 if the device is bound successfully and a positive + * (inverted) error code for failures from the ->probe method. * * This function must be called with @dev lock held. When called for a * USB interface, @dev->parent lock must be held as well. + * + * If the device has a parent, runtime-resume the parent before driver probing. */ -int driver_probe_device(struct device_driver *drv, struct device *dev) +static int driver_probe_device(const struct device_driver *drv, struct device *dev) { - int ret = 0; + int trigger_count = atomic_read(&deferred_trigger_count); + int ret; - if (!device_is_registered(dev)) - return -ENODEV; + atomic_inc(&probe_count); + ret = __driver_probe_device(drv, dev); + if (ret == -EPROBE_DEFER || ret == EPROBE_DEFER) { + driver_deferred_probe_add(dev); + + /* + * Did a trigger occur while probing? Need to re-trigger if yes + */ + if (trigger_count != atomic_read(&deferred_trigger_count) && + !defer_all_probes) + driver_deferred_probe_trigger(); + } + atomic_dec(&probe_count); + wake_up_all(&probe_waitqueue); + return ret; +} + +static inline bool cmdline_requested_async_probing(const char *drv_name) +{ + bool async_drv; - pr_debug("bus: '%s': %s: matched device %s with driver %s\n", - drv->bus->name, __func__, dev_name(dev), drv->name); + async_drv = parse_option_str(async_probe_drv_names, drv_name); - pm_runtime_barrier(dev); - ret = really_probe(dev, drv); - pm_request_idle(dev); + return (async_probe_default != async_drv); +} - return ret; +/* The option format is "driver_async_probe=drv_name1,drv_name2,..." */ +static int __init save_async_options(char *buf) +{ + if (strlen(buf) >= ASYNC_DRV_NAMES_MAX_LEN) + pr_warn("Too long list of driver names for 'driver_async_probe'!\n"); + + strscpy(async_probe_drv_names, buf, ASYNC_DRV_NAMES_MAX_LEN); + async_probe_default = parse_option_str(async_probe_drv_names, "*"); + + return 1; +} +__setup("driver_async_probe=", save_async_options); + +static bool driver_allows_async_probing(const struct device_driver *drv) +{ + switch (drv->probe_type) { + case PROBE_PREFER_ASYNCHRONOUS: + return true; + + case PROBE_FORCE_SYNCHRONOUS: + return false; + + default: + if (cmdline_requested_async_probing(drv->name)) + return true; + + if (module_requested_async_probing(drv->owner)) + return true; + + return false; + } } -static int __device_attach(struct device_driver *drv, void *data) +struct device_attach_data { + struct device *dev; + + /* + * Indicates whether we are considering asynchronous probing or + * not. Only initial binding after device or driver registration + * (including deferral processing) may be done asynchronously, the + * rest is always synchronous, as we expect it is being done by + * request from userspace. + */ + bool check_async; + + /* + * Indicates if we are binding synchronous or asynchronous drivers. + * When asynchronous probing is enabled we'll execute 2 passes + * over drivers: first pass doing synchronous probing and second + * doing asynchronous probing (if synchronous did not succeed - + * most likely because there was no driver requiring synchronous + * probing - and we found asynchronous driver during first pass). + * The 2 passes are done because we can't shoot asynchronous + * probe for given device and driver from bus_for_each_drv() since + * driver pointer is not guaranteed to stay valid once + * bus_for_each_drv() iterates to the next driver on the bus. + */ + bool want_async; + + /* + * We'll set have_async to 'true' if, while scanning for matching + * driver, we'll encounter one that requests asynchronous probing. + */ + bool have_async; +}; + +static int __device_attach_driver(struct device_driver *drv, void *_data) { - struct device *dev = data; + struct device_attach_data *data = _data; + struct device *dev = data->dev; + bool async_allowed; + int ret; + + ret = driver_match_device(drv, dev); + if (ret == 0) { + /* no match */ + return 0; + } else if (ret == -EPROBE_DEFER) { + dev_dbg(dev, "Device match requests probe deferral\n"); + dev->can_match = true; + driver_deferred_probe_add(dev); + /* + * Device can't match with a driver right now, so don't attempt + * to match or bind with other drivers on the bus. + */ + return ret; + } else if (ret < 0) { + dev_dbg(dev, "Bus failed to match device: %d\n", ret); + return ret; + } /* ret > 0 means positive match */ - if (!driver_match_device(drv, dev)) + async_allowed = driver_allows_async_probing(drv); + + if (async_allowed) + data->have_async = true; + + if (data->check_async && async_allowed != data->want_async) return 0; - return driver_probe_device(drv, dev); + /* + * Ignore errors returned by ->probe so that the next driver can try + * its luck. + */ + ret = driver_probe_device(drv, dev); + if (ret < 0) + return ret; + return ret == 0; +} + +static void __device_attach_async_helper(void *_dev, async_cookie_t cookie) +{ + struct device *dev = _dev; + struct device_attach_data data = { + .dev = dev, + .check_async = true, + .want_async = true, + }; + + device_lock(dev); + + /* + * Check if device has already been removed or claimed. This may + * happen with driver loading, device discovery/registration, + * and deferred probe processing happens all at once with + * multiple threads. + */ + if (dev->p->dead || dev->driver) + goto out_unlock; + + if (dev->parent) + pm_runtime_get_sync(dev->parent); + + bus_for_each_drv(dev->bus, NULL, &data, __device_attach_driver); + dev_dbg(dev, "async probe completed\n"); + + pm_request_idle(dev); + + if (dev->parent) + pm_runtime_put(dev->parent); +out_unlock: + device_unlock(dev); + + put_device(dev); +} + +static int __device_attach(struct device *dev, bool allow_async) +{ + int ret = 0; + bool async = false; + + device_lock(dev); + if (dev->p->dead) { + goto out_unlock; + } else if (dev->driver) { + if (device_is_bound(dev)) { + ret = 1; + goto out_unlock; + } + ret = device_bind_driver(dev); + if (ret == 0) + ret = 1; + else { + device_set_driver(dev, NULL); + ret = 0; + } + } else { + struct device_attach_data data = { + .dev = dev, + .check_async = allow_async, + .want_async = false, + }; + + if (dev->parent) + pm_runtime_get_sync(dev->parent); + + ret = bus_for_each_drv(dev->bus, NULL, &data, + __device_attach_driver); + if (!ret && allow_async && data.have_async) { + /* + * If we could not find appropriate driver + * synchronously and we are allowed to do + * async probes and there are drivers that + * want to probe asynchronously, we'll + * try them. + */ + dev_dbg(dev, "scheduling asynchronous probe\n"); + get_device(dev); + async = true; + } else { + pm_request_idle(dev); + } + + if (dev->parent) + pm_runtime_put(dev->parent); + } +out_unlock: + device_unlock(dev); + if (async) + async_schedule_dev(__device_attach_async_helper, dev); + return ret; } /** @@ -411,34 +1071,102 @@ static int __device_attach(struct device_driver *drv, void *data) */ int device_attach(struct device *dev) { - int ret = 0; + return __device_attach(dev, false); +} +EXPORT_SYMBOL_GPL(device_attach); + +void device_initial_probe(struct device *dev) +{ + struct subsys_private *sp = bus_to_subsys(dev->bus); + + if (!sp) + return; + if (sp->drivers_autoprobe) + __device_attach(dev, true); + + subsys_put(sp); +} + +/* + * __device_driver_lock - acquire locks needed to manipulate dev->drv + * @dev: Device we will update driver info for + * @parent: Parent device. Needed if the bus requires parent lock + * + * This function will take the required locks for manipulating dev->drv. + * Normally this will just be the @dev lock, but when called for a USB + * interface, @parent lock will be held as well. + */ +static void __device_driver_lock(struct device *dev, struct device *parent) +{ + if (parent && dev->bus->need_parent_lock) + device_lock(parent); device_lock(dev); - if (dev->driver) { - if (klist_node_attached(&dev->p->knode_driver)) { - ret = 1; - goto out_unlock; - } - ret = device_bind_driver(dev); - if (ret == 0) - ret = 1; - else { - dev->driver = NULL; - ret = 0; - } - } else { - ret = bus_for_each_drv(dev->bus, NULL, dev, __device_attach); - pm_request_idle(dev); - } -out_unlock: +} + +/* + * __device_driver_unlock - release locks needed to manipulate dev->drv + * @dev: Device we will update driver info for + * @parent: Parent device. Needed if the bus requires parent lock + * + * This function will release the required locks for manipulating dev->drv. + * Normally this will just be the @dev lock, but when called for a + * USB interface, @parent lock will be released as well. + */ +static void __device_driver_unlock(struct device *dev, struct device *parent) +{ device_unlock(dev); + if (parent && dev->bus->need_parent_lock) + device_unlock(parent); +} + +/** + * device_driver_attach - attach a specific driver to a specific device + * @drv: Driver to attach + * @dev: Device to attach it to + * + * Manually attach driver to a device. Will acquire both @dev lock and + * @dev->parent lock if needed. Returns 0 on success, -ERR on failure. + */ +int device_driver_attach(const struct device_driver *drv, struct device *dev) +{ + int ret; + + __device_driver_lock(dev, dev->parent); + ret = __driver_probe_device(drv, dev); + __device_driver_unlock(dev, dev->parent); + + /* also return probe errors as normal negative errnos */ + if (ret > 0) + ret = -ret; + if (ret == -EPROBE_DEFER) + return -EAGAIN; return ret; } -EXPORT_SYMBOL_GPL(device_attach); +EXPORT_SYMBOL_GPL(device_driver_attach); + +static void __driver_attach_async_helper(void *_dev, async_cookie_t cookie) +{ + struct device *dev = _dev; + const struct device_driver *drv; + int ret; + + __device_driver_lock(dev, dev->parent); + drv = dev->p->async_driver; + dev->p->async_driver = NULL; + ret = driver_probe_device(drv, dev); + __device_driver_unlock(dev, dev->parent); + + dev_dbg(dev, "driver %s async attach completed: %d\n", drv->name, ret); + + put_device(dev); +} static int __driver_attach(struct device *dev, void *data) { - struct device_driver *drv = data; + const struct device_driver *drv = data; + bool async = false; + int ret; /* * Lock device and try to bind to it. We drop the error @@ -450,17 +1178,52 @@ static int __driver_attach(struct device *dev, void *data) * is an error. */ - if (!driver_match_device(drv, dev)) + ret = driver_match_device(drv, dev); + if (ret == 0) { + /* no match */ + return 0; + } else if (ret == -EPROBE_DEFER) { + dev_dbg(dev, "Device match requests probe deferral\n"); + dev->can_match = true; + driver_deferred_probe_add(dev); + /* + * Driver could not match with device, but may match with + * another device on the bus. + */ + return 0; + } else if (ret < 0) { + dev_dbg(dev, "Bus failed to match device: %d\n", ret); + /* + * Driver could not match with device, but may match with + * another device on the bus. + */ return 0; + } /* ret > 0 means positive match */ - if (dev->parent) /* Needed for USB */ - device_lock(dev->parent); - device_lock(dev); - if (!dev->driver) - driver_probe_device(drv, dev); - device_unlock(dev); - if (dev->parent) - device_unlock(dev->parent); + if (driver_allows_async_probing(drv)) { + /* + * Instead of probing the device synchronously we will + * probe it asynchronously to allow for more parallelism. + * + * We only take the device lock here in order to guarantee + * that the dev->driver and async_driver fields are protected + */ + dev_dbg(dev, "probing driver %s asynchronously\n", drv->name); + device_lock(dev); + if (!dev->driver && !dev->p->async_driver) { + get_device(dev); + dev->p->async_driver = drv; + async = true; + } + device_unlock(dev); + if (async) + async_schedule_dev(__driver_attach_async_helper, dev); + return 0; + } + + __device_driver_lock(dev, dev->parent); + driver_probe_device(drv, dev); + __device_driver_unlock(dev, dev->parent); return 0; } @@ -474,9 +1237,10 @@ static int __driver_attach(struct device *dev, void *data) * returns 0 and the @dev->driver is set, we've found a * compatible pair. */ -int driver_attach(struct device_driver *drv) +int driver_attach(const struct device_driver *drv) { - return bus_for_each_dev(drv->bus, NULL, drv, __driver_attach); + /* The (void *) will be put back to const * in __driver_attach() */ + return bus_for_each_dev(drv->bus, NULL, (void *)drv, __driver_attach); } EXPORT_SYMBOL_GPL(driver_attach); @@ -484,7 +1248,7 @@ EXPORT_SYMBOL_GPL(driver_attach); * __device_release_driver() must be called with @dev lock held. * When called for a USB interface, @dev->parent lock must be held as well. */ -static void __device_release_driver(struct device *dev) +static void __device_release_driver(struct device *dev, struct device *parent) { struct device_driver *drv; @@ -492,37 +1256,67 @@ static void __device_release_driver(struct device *dev) if (drv) { pm_runtime_get_sync(dev); + while (device_links_busy(dev)) { + __device_driver_unlock(dev, parent); + + device_links_unbind_consumers(dev); + + __device_driver_lock(dev, parent); + /* + * A concurrent invocation of the same function might + * have released the driver successfully while this one + * was waiting, so check for that. + */ + if (dev->driver != drv) { + pm_runtime_put(dev); + return; + } + } + driver_sysfs_remove(dev); - if (dev->bus) - blocking_notifier_call_chain(&dev->bus->p->bus_notifier, - BUS_NOTIFY_UNBIND_DRIVER, - dev); + bus_notify(dev, BUS_NOTIFY_UNBIND_DRIVER); + + pm_runtime_put_sync(dev); + + device_remove(dev); + + if (dev->bus && dev->bus->dma_cleanup) + dev->bus->dma_cleanup(dev); - pm_runtime_put(dev); + device_unbind_cleanup(dev); + device_links_driver_cleanup(dev); - if (dev->bus && dev->bus->remove) - dev->bus->remove(dev); - else if (drv->remove) - drv->remove(dev); - devres_release_all(dev); - dev->driver = NULL; - dev_set_drvdata(dev, NULL); klist_remove(&dev->p->knode_driver); - if (dev->bus) - blocking_notifier_call_chain(&dev->bus->p->bus_notifier, - BUS_NOTIFY_UNBOUND_DRIVER, - dev); + device_pm_check_callbacks(dev); + bus_notify(dev, BUS_NOTIFY_UNBOUND_DRIVER); + kobject_uevent(&dev->kobj, KOBJ_UNBIND); } } +void device_release_driver_internal(struct device *dev, + const struct device_driver *drv, + struct device *parent) +{ + __device_driver_lock(dev, parent); + + if (!drv || drv == dev->driver) + __device_release_driver(dev, parent); + + __device_driver_unlock(dev, parent); +} + /** * device_release_driver - manually detach device from driver. * @dev: device. * * Manually detach device from driver. * When called for a USB interface, @dev->parent lock must be held. + * + * If this function is to be called with @dev->parent lock held, ensure that + * the device's consumers are unbound in advance or that their locks can be + * acquired under the @dev->parent lock. */ void device_release_driver(struct device *dev) { @@ -531,68 +1325,47 @@ void device_release_driver(struct device *dev) * within their ->remove callback for the same device, they * will deadlock right here. */ - device_lock(dev); - __device_release_driver(dev); - device_unlock(dev); + device_release_driver_internal(dev, NULL, NULL); } EXPORT_SYMBOL_GPL(device_release_driver); /** + * device_driver_detach - detach driver from a specific device + * @dev: device to detach driver from + * + * Detach driver from device. Will acquire both @dev lock and @dev->parent + * lock if needed. + */ +void device_driver_detach(struct device *dev) +{ + device_release_driver_internal(dev, NULL, dev->parent); +} + +/** * driver_detach - detach driver from all devices it controls. * @drv: driver. */ -void driver_detach(struct device_driver *drv) +void driver_detach(const struct device_driver *drv) { struct device_private *dev_prv; struct device *dev; + if (driver_allows_async_probing(drv)) + async_synchronize_full(); + for (;;) { spin_lock(&drv->p->klist_devices.k_lock); if (list_empty(&drv->p->klist_devices.k_list)) { spin_unlock(&drv->p->klist_devices.k_lock); break; } - dev_prv = list_entry(drv->p->klist_devices.k_list.prev, + dev_prv = list_last_entry(&drv->p->klist_devices.k_list, struct device_private, knode_driver.n_node); dev = dev_prv->device; get_device(dev); spin_unlock(&drv->p->klist_devices.k_lock); - - if (dev->parent) /* Needed for USB */ - device_lock(dev->parent); - device_lock(dev); - if (dev->driver == drv) - __device_release_driver(dev); - device_unlock(dev); - if (dev->parent) - device_unlock(dev->parent); + device_release_driver_internal(dev, drv, dev->parent); put_device(dev); } } - -/* - * These exports can't be _GPL due to .h files using this within them, and it - * might break something that was previously working... - */ -void *dev_get_drvdata(const struct device *dev) -{ - if (dev && dev->p) - return dev->p->driver_data; - return NULL; -} -EXPORT_SYMBOL(dev_get_drvdata); - -int dev_set_drvdata(struct device *dev, void *data) -{ - int error; - - if (!dev->p) { - error = device_private_init(dev); - if (error) - return error; - } - dev->p->driver_data = data; - return 0; -} -EXPORT_SYMBOL(dev_set_drvdata); diff --git a/drivers/base/devcoredump.c b/drivers/base/devcoredump.c new file mode 100644 index 000000000000..55bdc7f5e59d --- /dev/null +++ b/drivers/base/devcoredump.c @@ -0,0 +1,480 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright(c) 2014 Intel Mobile Communications GmbH + * Copyright(c) 2015 Intel Deutschland GmbH + * + * Author: Johannes Berg <johannes@sipsolutions.net> + */ +#include <linux/module.h> +#include <linux/device.h> +#include <linux/devcoredump.h> +#include <linux/list.h> +#include <linux/slab.h> +#include <linux/fs.h> +#include <linux/workqueue.h> + +static struct class devcd_class; + +/* global disable flag, for security purposes */ +static bool devcd_disabled; + +struct devcd_entry { + struct device devcd_dev; + void *data; + size_t datalen; + /* + * There are 2 races for which mutex is required. + * + * The first race is between device creation and userspace writing to + * schedule immediately destruction. + * + * This race is handled by arming the timer before device creation, but + * when device creation fails the timer still exists. + * + * To solve this, hold the mutex during device_add(), and set + * init_completed on success before releasing the mutex. + * + * That way the timer will never fire until device_add() is called, + * it will do nothing if init_completed is not set. The timer is also + * cancelled in that case. + * + * The second race involves multiple parallel invocations of devcd_free(), + * add a deleted flag so only 1 can call the destructor. + */ + struct mutex mutex; + bool init_completed, deleted; + struct module *owner; + ssize_t (*read)(char *buffer, loff_t offset, size_t count, + void *data, size_t datalen); + void (*free)(void *data); + /* + * If nothing interferes and device_add() was returns success, + * del_wk will destroy the device after the timer fires. + * + * Multiple userspace processes can interfere in the working of the timer: + * - Writing to the coredump will reschedule the timer to run immediately, + * if still armed. + * + * This is handled by using "if (cancel_delayed_work()) { + * schedule_delayed_work() }", to prevent re-arming after having + * been previously fired. + * - Writing to /sys/class/devcoredump/disabled will destroy the + * coredump synchronously. + * This is handled by using disable_delayed_work_sync(), and then + * checking if deleted flag is set with &devcd->mutex held. + */ + struct delayed_work del_wk; + struct device *failing_dev; +}; + +static struct devcd_entry *dev_to_devcd(struct device *dev) +{ + return container_of(dev, struct devcd_entry, devcd_dev); +} + +static void devcd_dev_release(struct device *dev) +{ + struct devcd_entry *devcd = dev_to_devcd(dev); + + devcd->free(devcd->data); + module_put(devcd->owner); + + /* + * this seems racy, but I don't see a notifier or such on + * a struct device to know when it goes away? + */ + if (devcd->failing_dev->kobj.sd) + sysfs_delete_link(&devcd->failing_dev->kobj, &dev->kobj, + "devcoredump"); + + put_device(devcd->failing_dev); + kfree(devcd); +} + +static void __devcd_del(struct devcd_entry *devcd) +{ + devcd->deleted = true; + device_del(&devcd->devcd_dev); + put_device(&devcd->devcd_dev); +} + +static void devcd_del(struct work_struct *wk) +{ + struct devcd_entry *devcd; + bool init_completed; + + devcd = container_of(wk, struct devcd_entry, del_wk.work); + + /* devcd->mutex serializes against dev_coredumpm_timeout */ + mutex_lock(&devcd->mutex); + init_completed = devcd->init_completed; + mutex_unlock(&devcd->mutex); + + if (init_completed) + __devcd_del(devcd); +} + +static ssize_t devcd_data_read(struct file *filp, struct kobject *kobj, + const struct bin_attribute *bin_attr, + char *buffer, loff_t offset, size_t count) +{ + struct device *dev = kobj_to_dev(kobj); + struct devcd_entry *devcd = dev_to_devcd(dev); + + return devcd->read(buffer, offset, count, devcd->data, devcd->datalen); +} + +static ssize_t devcd_data_write(struct file *filp, struct kobject *kobj, + const struct bin_attribute *bin_attr, + char *buffer, loff_t offset, size_t count) +{ + struct device *dev = kobj_to_dev(kobj); + struct devcd_entry *devcd = dev_to_devcd(dev); + + /* + * Although it's tempting to use mod_delayed work here, + * that will cause a reschedule if the timer already fired. + */ + if (cancel_delayed_work(&devcd->del_wk)) + schedule_delayed_work(&devcd->del_wk, 0); + + return count; +} + +static const struct bin_attribute devcd_attr_data = + __BIN_ATTR(data, 0600, devcd_data_read, devcd_data_write, 0); + +static const struct bin_attribute *const devcd_dev_bin_attrs[] = { + &devcd_attr_data, NULL, +}; + +static const struct attribute_group devcd_dev_group = { + .bin_attrs = devcd_dev_bin_attrs, +}; + +static const struct attribute_group *devcd_dev_groups[] = { + &devcd_dev_group, NULL, +}; + +static int devcd_free(struct device *dev, void *data) +{ + struct devcd_entry *devcd = dev_to_devcd(dev); + + /* + * To prevent a race with devcd_data_write(), disable work and + * complete manually instead. + * + * We cannot rely on the return value of + * disable_delayed_work_sync() here, because it might be in the + * middle of a cancel_delayed_work + schedule_delayed_work pair. + * + * devcd->mutex here guards against multiple parallel invocations + * of devcd_free(). + */ + disable_delayed_work_sync(&devcd->del_wk); + mutex_lock(&devcd->mutex); + if (!devcd->deleted) + __devcd_del(devcd); + mutex_unlock(&devcd->mutex); + return 0; +} + +static ssize_t disabled_show(const struct class *class, const struct class_attribute *attr, + char *buf) +{ + return sysfs_emit(buf, "%d\n", devcd_disabled); +} + +/* + * + * disabled_store() worker() + * class_for_each_device(&devcd_class, + * NULL, NULL, devcd_free) + * ... + * ... + * while ((dev = class_dev_iter_next(&iter)) + * devcd_del() + * device_del() + * put_device() <- last reference + * error = fn(dev, data) devcd_dev_release() + * devcd_free(dev, data) kfree(devcd) + * + * + * In the above diagram, it looks like disabled_store() would be racing with parallelly + * running devcd_del() and result in memory abort after dropping its last reference with + * put_device(). However, this will not happens as fn(dev, data) runs + * with its own reference to device via klist_node so it is not its last reference. + * so, above situation would not occur. + */ + +static ssize_t disabled_store(const struct class *class, const struct class_attribute *attr, + const char *buf, size_t count) +{ + long tmp = simple_strtol(buf, NULL, 10); + + /* + * This essentially makes the attribute write-once, since you can't + * go back to not having it disabled. This is intentional, it serves + * as a system lockdown feature. + */ + if (tmp != 1) + return -EINVAL; + + devcd_disabled = true; + + class_for_each_device(&devcd_class, NULL, NULL, devcd_free); + + return count; +} +static CLASS_ATTR_RW(disabled); + +static struct attribute *devcd_class_attrs[] = { + &class_attr_disabled.attr, + NULL, +}; +ATTRIBUTE_GROUPS(devcd_class); + +static struct class devcd_class = { + .name = "devcoredump", + .dev_release = devcd_dev_release, + .dev_groups = devcd_dev_groups, + .class_groups = devcd_class_groups, +}; + +static ssize_t devcd_readv(char *buffer, loff_t offset, size_t count, + void *data, size_t datalen) +{ + return memory_read_from_buffer(buffer, count, &offset, data, datalen); +} + +static void devcd_freev(void *data) +{ + vfree(data); +} + +/** + * dev_coredumpv - create device coredump with vmalloc data + * @dev: the struct device for the crashed device + * @data: vmalloc data containing the device coredump + * @datalen: length of the data + * @gfp: allocation flags + * + * This function takes ownership of the vmalloc'ed data and will free + * it when it is no longer used. See dev_coredumpm() for more information. + */ +void dev_coredumpv(struct device *dev, void *data, size_t datalen, + gfp_t gfp) +{ + dev_coredumpm(dev, NULL, data, datalen, gfp, devcd_readv, devcd_freev); +} +EXPORT_SYMBOL_GPL(dev_coredumpv); + +static int devcd_match_failing(struct device *dev, const void *failing) +{ + struct devcd_entry *devcd = dev_to_devcd(dev); + + return devcd->failing_dev == failing; +} + +/** + * devcd_free_sgtable - free all the memory of the given scatterlist table + * (i.e. both pages and scatterlist instances) + * NOTE: if two tables allocated with devcd_alloc_sgtable and then chained + * using the sg_chain function then that function should be called only once + * on the chained table + * @data: pointer to sg_table to free + */ +static void devcd_free_sgtable(void *data) +{ + _devcd_free_sgtable(data); +} + +/** + * devcd_read_from_sgtable - copy data from sg_table to a given buffer + * and return the number of bytes read + * @buffer: the buffer to copy the data to it + * @buf_len: the length of the buffer + * @data: the scatterlist table to copy from + * @offset: start copy from @offset@ bytes from the head of the data + * in the given scatterlist + * @data_len: the length of the data in the sg_table + * + * Returns: the number of bytes copied + */ +static ssize_t devcd_read_from_sgtable(char *buffer, loff_t offset, + size_t buf_len, void *data, + size_t data_len) +{ + struct scatterlist *table = data; + + if (offset > data_len) + return -EINVAL; + + if (offset + buf_len > data_len) + buf_len = data_len - offset; + return sg_pcopy_to_buffer(table, sg_nents(table), buffer, buf_len, + offset); +} + +/** + * dev_coredump_put - remove device coredump + * @dev: the struct device for the crashed device + * + * dev_coredump_put() removes coredump, if exists, for a given device from + * the file system and free its associated data otherwise, does nothing. + * + * It is useful for modules that do not want to keep coredump + * available after its unload. + */ +void dev_coredump_put(struct device *dev) +{ + struct device *existing; + + existing = class_find_device(&devcd_class, NULL, dev, + devcd_match_failing); + if (existing) { + devcd_free(existing, NULL); + put_device(existing); + } +} +EXPORT_SYMBOL_GPL(dev_coredump_put); + +/** + * dev_coredumpm_timeout - create device coredump with read/free methods with a + * custom timeout. + * @dev: the struct device for the crashed device + * @owner: the module that contains the read/free functions, use %THIS_MODULE + * @data: data cookie for the @read/@free functions + * @datalen: length of the data + * @gfp: allocation flags + * @read: function to read from the given buffer + * @free: function to free the given buffer + * @timeout: time in jiffies to remove coredump + * + * Creates a new device coredump for the given device. If a previous one hasn't + * been read yet, the new coredump is discarded. The data lifetime is determined + * by the device coredump framework and when it is no longer needed the @free + * function will be called to free the data. + */ +void dev_coredumpm_timeout(struct device *dev, struct module *owner, + void *data, size_t datalen, gfp_t gfp, + ssize_t (*read)(char *buffer, loff_t offset, + size_t count, void *data, + size_t datalen), + void (*free)(void *data), + unsigned long timeout) +{ + static atomic_t devcd_count = ATOMIC_INIT(0); + struct devcd_entry *devcd; + struct device *existing; + + if (devcd_disabled) + goto free; + + existing = class_find_device(&devcd_class, NULL, dev, + devcd_match_failing); + if (existing) { + put_device(existing); + goto free; + } + + if (!try_module_get(owner)) + goto free; + + devcd = kzalloc(sizeof(*devcd), gfp); + if (!devcd) + goto put_module; + + devcd->owner = owner; + devcd->data = data; + devcd->datalen = datalen; + devcd->read = read; + devcd->free = free; + devcd->failing_dev = get_device(dev); + devcd->deleted = false; + + mutex_init(&devcd->mutex); + device_initialize(&devcd->devcd_dev); + + dev_set_name(&devcd->devcd_dev, "devcd%d", + atomic_inc_return(&devcd_count)); + devcd->devcd_dev.class = &devcd_class; + + dev_set_uevent_suppress(&devcd->devcd_dev, true); + + /* devcd->mutex prevents devcd_del() completing until init finishes */ + mutex_lock(&devcd->mutex); + devcd->init_completed = false; + INIT_DELAYED_WORK(&devcd->del_wk, devcd_del); + schedule_delayed_work(&devcd->del_wk, timeout); + + if (device_add(&devcd->devcd_dev)) + goto put_device; + + /* + * These should normally not fail, but there is no problem + * continuing without the links, so just warn instead of + * failing. + */ + if (sysfs_create_link(&devcd->devcd_dev.kobj, &dev->kobj, + "failing_device") || + sysfs_create_link(&dev->kobj, &devcd->devcd_dev.kobj, + "devcoredump")) + dev_warn(dev, "devcoredump create_link failed\n"); + + dev_set_uevent_suppress(&devcd->devcd_dev, false); + kobject_uevent(&devcd->devcd_dev.kobj, KOBJ_ADD); + + /* + * Safe to run devcd_del() now that we are done with devcd_dev. + * Alternatively we could have taken a ref on devcd_dev before + * dropping the lock. + */ + devcd->init_completed = true; + mutex_unlock(&devcd->mutex); + return; + put_device: + mutex_unlock(&devcd->mutex); + cancel_delayed_work_sync(&devcd->del_wk); + put_device(&devcd->devcd_dev); + + put_module: + module_put(owner); + free: + free(data); +} +EXPORT_SYMBOL_GPL(dev_coredumpm_timeout); + +/** + * dev_coredumpsg - create device coredump that uses scatterlist as data + * parameter + * @dev: the struct device for the crashed device + * @table: the dump data + * @datalen: length of the data + * @gfp: allocation flags + * + * Creates a new device coredump for the given device. If a previous one hasn't + * been read yet, the new coredump is discarded. The data lifetime is determined + * by the device coredump framework and when it is no longer needed + * it will free the data. + */ +void dev_coredumpsg(struct device *dev, struct scatterlist *table, + size_t datalen, gfp_t gfp) +{ + dev_coredumpm(dev, NULL, table, datalen, gfp, devcd_read_from_sgtable, + devcd_free_sgtable); +} +EXPORT_SYMBOL_GPL(dev_coredumpsg); + +static int __init devcoredump_init(void) +{ + return class_register(&devcd_class); +} +__initcall(devcoredump_init); + +static void __exit devcoredump_exit(void) +{ + class_for_each_device(&devcd_class, NULL, NULL, devcd_free); + class_unregister(&devcd_class); +} +__exitcall(devcoredump_exit); diff --git a/drivers/base/devres.c b/drivers/base/devres.c index 507379e7b763..f54db6d138ab 100644 --- a/drivers/base/devres.c +++ b/drivers/base/devres.c @@ -1,31 +1,38 @@ +// SPDX-License-Identifier: GPL-2.0 /* * drivers/base/devres.c - device resource management * * Copyright (c) 2006 SUSE Linux Products GmbH * Copyright (c) 2006 Tejun Heo <teheo@suse.de> - * - * This file is released under the GPLv2. */ #include <linux/device.h> #include <linux/module.h> #include <linux/slab.h> +#include <linux/percpu.h> + +#include <asm/sections.h> #include "base.h" +#include "trace.h" struct devres_node { struct list_head entry; dr_release_t release; -#ifdef CONFIG_DEBUG_DEVRES const char *name; size_t size; -#endif }; struct devres { struct devres_node node; - /* -- 3 pointers */ - unsigned long long data[]; /* guarantee ull alignment */ + /* + * Some archs want to perform DMA into kmalloc caches + * and need a guaranteed alignment larger than + * the alignment of a 64-bit integer. + * Thus we use ARCH_DMA_MINALIGN for data[] which will force the same + * alignment for struct devres when allocated by kmalloc(). + */ + u8 __aligned(ARCH_DMA_MINALIGN) data[]; }; struct devres_group { @@ -35,10 +42,6 @@ struct devres_group { /* -- 8 pointers */ }; -#ifdef CONFIG_DEBUG_DEVRES -static int log_devres = 0; -module_param_named(log, log_devres, int, S_IRUGO | S_IWUSR); - static void set_node_dbginfo(struct devres_node *node, const char *name, size_t size) { @@ -46,18 +49,28 @@ static void set_node_dbginfo(struct devres_node *node, const char *name, node->size = size; } -static void devres_log(struct device *dev, struct devres_node *node, +#ifdef CONFIG_DEBUG_DEVRES +static int log_devres = 0; +module_param_named(log, log_devres, int, S_IRUGO | S_IWUSR); + +static void devres_dbg(struct device *dev, struct devres_node *node, const char *op) { if (unlikely(log_devres)) - dev_err(dev, "DEVRES %3s %p %s (%lu bytes)\n", - op, node, node->name, (unsigned long)node->size); + dev_err(dev, "DEVRES %3s %p %s (%zu bytes)\n", + op, node, node->name, node->size); } #else /* CONFIG_DEBUG_DEVRES */ -#define set_node_dbginfo(node, n, s) do {} while (0) -#define devres_log(dev, node, op) do {} while (0) +#define devres_dbg(dev, node, op) do {} while (0) #endif /* CONFIG_DEBUG_DEVRES */ +static void devres_log(struct device *dev, struct devres_node *node, + const char *op) +{ + trace_devres_log(dev, op, node, node->name, node->size); + devres_dbg(dev, node, op); +} + /* * Release functions for devres group. These callbacks are used only * for identification. @@ -72,7 +85,7 @@ static void group_close_release(struct device *dev, void *res) /* noop */ } -static struct devres_group * node_to_group(struct devres_node *node) +static struct devres_group *node_to_group(struct devres_node *node) { if (node->release == &group_open_release) return container_of(node, struct devres_group, node[0]); @@ -81,17 +94,36 @@ static struct devres_group * node_to_group(struct devres_node *node) return NULL; } -static __always_inline struct devres * alloc_dr(dr_release_t release, - size_t size, gfp_t gfp) +static bool check_dr_size(size_t size, size_t *tot_size) +{ + /* We must catch any near-SIZE_MAX cases that could overflow. */ + if (unlikely(check_add_overflow(sizeof(struct devres), + size, tot_size))) + return false; + + /* Actually allocate the full kmalloc bucket size. */ + *tot_size = kmalloc_size_roundup(*tot_size); + + return true; +} + +static __always_inline struct devres *alloc_dr(dr_release_t release, + size_t size, gfp_t gfp, int nid) { - size_t tot_size = sizeof(struct devres) + size; + size_t tot_size; struct devres *dr; - dr = kmalloc_track_caller(tot_size, gfp); + if (!check_dr_size(size, &tot_size)) + return NULL; + + dr = kmalloc_node_track_caller(tot_size, gfp, nid); if (unlikely(!dr)) return NULL; - memset(dr, 0, tot_size); + /* No need to clear memory twice */ + if (!(gfp & __GFP_ZERO)) + memset(dr, 0, offsetof(struct devres, data)); + INIT_LIST_HEAD(&dr->node.entry); dr->node.release = release; return dr; @@ -104,25 +136,21 @@ static void add_dr(struct device *dev, struct devres_node *node) list_add_tail(&node->entry, &dev->devres_head); } -#ifdef CONFIG_DEBUG_DEVRES -void * __devres_alloc(dr_release_t release, size_t size, gfp_t gfp, - const char *name) +static void replace_dr(struct device *dev, + struct devres_node *old, struct devres_node *new) { - struct devres *dr; - - dr = alloc_dr(release, size, gfp); - if (unlikely(!dr)) - return NULL; - set_node_dbginfo(&dr->node, name, size); - return dr->data; + devres_log(dev, old, "REPLACE"); + BUG_ON(!list_empty(&new->entry)); + list_replace(&old->entry, &new->entry); } -EXPORT_SYMBOL_GPL(__devres_alloc); -#else + /** - * devres_alloc - Allocate device resource data + * __devres_alloc_node - Allocate device resource data * @release: Release function devres will be associated with * @size: Allocation size * @gfp: Allocation flags + * @nid: NUMA node + * @name: Name of the resource * * Allocate devres of @size bytes. The allocated area is zeroed, then * associated with @release. The returned pointer can be passed to @@ -131,17 +159,18 @@ EXPORT_SYMBOL_GPL(__devres_alloc); * RETURNS: * Pointer to allocated devres on success, NULL on failure. */ -void * devres_alloc(dr_release_t release, size_t size, gfp_t gfp) +void *__devres_alloc_node(dr_release_t release, size_t size, gfp_t gfp, int nid, + const char *name) { struct devres *dr; - dr = alloc_dr(release, size, gfp); + dr = alloc_dr(release, size, gfp | __GFP_ZERO, nid); if (unlikely(!dr)) return NULL; + set_node_dbginfo(&dr->node, name, size); return dr->data; } -EXPORT_SYMBOL_GPL(devres_alloc); -#endif +EXPORT_SYMBOL_GPL(__devres_alloc_node); /** * devres_for_each_res - Resource iterator @@ -254,8 +283,8 @@ static struct devres *find_dr(struct device *dev, dr_release_t release, * RETURNS: * Pointer to found devres, NULL if not found. */ -void * devres_find(struct device *dev, dr_release_t release, - dr_match_t match, void *match_data) +void *devres_find(struct device *dev, dr_release_t release, + dr_match_t match, void *match_data) { struct devres *dr; unsigned long flags; @@ -284,8 +313,8 @@ EXPORT_SYMBOL_GPL(devres_find); * RETURNS: * Pointer to found or added devres. */ -void * devres_get(struct device *dev, void *new_res, - dr_match_t match, void *match_data) +void *devres_get(struct device *dev, void *new_res, + dr_match_t match, void *match_data) { struct devres *new_dr = container_of(new_res, struct devres, data); struct devres *dr; @@ -296,10 +325,10 @@ void * devres_get(struct device *dev, void *new_res, if (!dr) { add_dr(dev, &new_dr->node); dr = new_dr; - new_dr = NULL; + new_res = NULL; } spin_unlock_irqrestore(&dev->devres_lock, flags); - devres_free(new_dr); + devres_free(new_res); return dr->data; } @@ -320,8 +349,8 @@ EXPORT_SYMBOL_GPL(devres_get); * RETURNS: * Pointer to removed devres on success, NULL if not found. */ -void * devres_remove(struct device *dev, dr_release_t release, - dr_match_t match, void *match_data) +void *devres_remove(struct device *dev, dr_release_t release, + dr_match_t match, void *match_data) { struct devres *dr; unsigned long flags; @@ -407,20 +436,16 @@ static int remove_nodes(struct device *dev, struct list_head *first, struct list_head *end, struct list_head *todo) { + struct devres_node *node, *n; int cnt = 0, nr_groups = 0; - struct list_head *cur; /* First pass - move normal devres entries to @todo and clear * devres_group colors. */ - cur = first; - while (cur != end) { - struct devres_node *node; + node = list_entry(first, struct devres_node, entry); + list_for_each_entry_safe_from(node, n, end, entry) { struct devres_group *grp; - node = list_entry(cur, struct devres_node, entry); - cur = cur->next; - grp = node_to_group(node); if (grp) { /* clear color of group markers in the first pass */ @@ -440,18 +465,14 @@ static int remove_nodes(struct device *dev, /* Second pass - Scan groups and color them. A group gets * color value of two iff the group is wholly contained in - * [cur, end). That is, for a closed group, both opening and - * closing markers should be in the range, while just the + * [current node, end). That is, for a closed group, both opening + * and closing markers should be in the range, while just the * opening marker is enough for an open group. */ - cur = first; - while (cur != end) { - struct devres_node *node; + node = list_entry(first, struct devres_node, entry); + list_for_each_entry_safe_from(node, n, end, entry) { struct devres_group *grp; - node = list_entry(cur, struct devres_node, entry); - cur = cur->next; - grp = node_to_group(node); BUG_ON(!grp || list_empty(&grp->node[0].entry)); @@ -461,7 +482,7 @@ static int remove_nodes(struct device *dev, BUG_ON(grp->color <= 0 || grp->color > 2); if (grp->color == 2) { - /* No need to update cur or end. The removed + /* No need to update current node or end. The removed * nodes are always before both. */ list_move_tail(&grp->node[0].entry, todo); @@ -472,28 +493,18 @@ static int remove_nodes(struct device *dev, return cnt; } -static int release_nodes(struct device *dev, struct list_head *first, - struct list_head *end, unsigned long flags) - __releases(&dev->devres_lock) +static void release_nodes(struct device *dev, struct list_head *todo) { - LIST_HEAD(todo); - int cnt; struct devres *dr, *tmp; - cnt = remove_nodes(dev, first, end, &todo); - - spin_unlock_irqrestore(&dev->devres_lock, flags); - /* Release. Note that both devres and devres_group are * handled as devres in the following loop. This is safe. */ - list_for_each_entry_safe_reverse(dr, tmp, &todo, node.entry) { + list_for_each_entry_safe_reverse(dr, tmp, todo, node.entry) { devres_log(dev, &dr->node, "REL"); dr->node.release(dev, dr->data); kfree(dr); } - - return cnt; } /** @@ -506,13 +517,23 @@ static int release_nodes(struct device *dev, struct list_head *first, int devres_release_all(struct device *dev) { unsigned long flags; + LIST_HEAD(todo); + int cnt; /* Looks like an uninitialized device structure */ if (WARN_ON(dev->devres_head.next == NULL)) return -ENODEV; + + /* Nothing to release if list is empty */ + if (list_empty(&dev->devres_head)) + return 0; + spin_lock_irqsave(&dev->devres_lock, flags); - return release_nodes(dev, dev->devres_head.next, &dev->devres_head, - flags); + cnt = remove_nodes(dev, dev->devres_head.next, &dev->devres_head, &todo); + spin_unlock_irqrestore(&dev->devres_lock, flags); + + release_nodes(dev, &todo); + return cnt; } /** @@ -528,7 +549,7 @@ int devres_release_all(struct device *dev) * RETURNS: * ID of the new group, NULL on failure. */ -void * devres_open_group(struct device *dev, void *id, gfp_t gfp) +void *devres_open_group(struct device *dev, void *id, gfp_t gfp) { struct devres_group *grp; unsigned long flags; @@ -546,6 +567,7 @@ void * devres_open_group(struct device *dev, void *id, gfp_t gfp) grp->id = grp; if (id) grp->id = id; + grp->color = 0; spin_lock_irqsave(&dev->devres_lock, flags); add_dr(dev, &grp->node[0]); @@ -554,8 +576,11 @@ void * devres_open_group(struct device *dev, void *id, gfp_t gfp) } EXPORT_SYMBOL_GPL(devres_open_group); -/* Find devres group with ID @id. If @id is NULL, look for the latest. */ -static struct devres_group * find_group(struct device *dev, void *id) +/* + * Find devres group with ID @id. If @id is NULL, look for the latest open + * group. + */ +static struct devres_group *find_group(struct device *dev, void *id) { struct devres_node *node; @@ -648,6 +673,7 @@ int devres_release_group(struct device *dev, void *id) { struct devres_group *grp; unsigned long flags; + LIST_HEAD(todo); int cnt = 0; spin_lock_irqsave(&dev->devres_lock, flags); @@ -660,7 +686,17 @@ int devres_release_group(struct device *dev, void *id) if (!list_empty(&grp->node[1].entry)) end = grp->node[1].entry.next; - cnt = release_nodes(dev, first, end, flags); + cnt = remove_nodes(dev, first, end, &todo); + spin_unlock_irqrestore(&dev->devres_lock, flags); + + release_nodes(dev, &todo); + } else if (list_empty(&dev->devres_head)) { + /* + * dev is probably dying via devres_release_all(): groups + * have already been removed and are on the process of + * being released - don't touch and don't warn. + */ + spin_unlock_irqrestore(&dev->devres_lock, flags); } else { WARN_ON(1); spin_unlock_irqrestore(&dev->devres_lock, flags); @@ -672,7 +708,7 @@ EXPORT_SYMBOL_GPL(devres_release_group); /* * Custom devres actions allow inserting a simple function call - * into the teadown sequence. + * into the teardown sequence. */ struct action_devres { @@ -697,20 +733,21 @@ static void devm_action_release(struct device *dev, void *res) } /** - * devm_add_action() - add a custom action to list of managed resources + * __devm_add_action() - add a custom action to list of managed resources * @dev: Device that owns the action * @action: Function that should be called * @data: Pointer to data passed to @action implementation + * @name: Name of the resource (for debugging purposes) * * This adds a custom action to the list of managed resources so that * it gets executed as part of standard resource unwinding. */ -int devm_add_action(struct device *dev, void (*action)(void *), void *data) +int __devm_add_action(struct device *dev, void (*action)(void *), void *data, const char *name) { struct action_devres *devres; - devres = devres_alloc(devm_action_release, - sizeof(struct action_devres), GFP_KERNEL); + devres = __devres_alloc_node(devm_action_release, sizeof(struct action_devres), + GFP_KERNEL, NUMA_NO_NODE, name); if (!devres) return -ENOMEM; @@ -720,83 +757,503 @@ int devm_add_action(struct device *dev, void (*action)(void *), void *data) devres_add(dev, devres); return 0; } -EXPORT_SYMBOL_GPL(devm_add_action); +EXPORT_SYMBOL_GPL(__devm_add_action); + +bool devm_is_action_added(struct device *dev, void (*action)(void *), void *data) +{ + struct action_devres devres = { + .data = data, + .action = action, + }; + + return devres_find(dev, devm_action_release, devm_action_match, &devres); +} +EXPORT_SYMBOL_GPL(devm_is_action_added); /** - * devm_remove_action() - removes previously added custom action + * devm_remove_action_nowarn() - removes previously added custom action * @dev: Device that owns the action * @action: Function implementing the action * @data: Pointer to data passed to @action implementation * * Removes instance of @action previously added by devm_add_action(). * Both action and data should match one of the existing entries. + * + * In contrast to devm_remove_action(), this function does not WARN() if no + * entry could have been found. + * + * This should only be used if the action is contained in an object with + * independent lifetime management, e.g. the Devres rust abstraction. + * + * Causing the warning from regular driver code most likely indicates an abuse + * of the devres API. + * + * Returns: 0 on success, -ENOENT if no entry could have been found. + */ +int devm_remove_action_nowarn(struct device *dev, + void (*action)(void *), + void *data) +{ + struct action_devres devres = { + .data = data, + .action = action, + }; + + return devres_destroy(dev, devm_action_release, devm_action_match, + &devres); +} +EXPORT_SYMBOL_GPL(devm_remove_action_nowarn); + +/** + * devm_release_action() - release previously added custom action + * @dev: Device that owns the action + * @action: Function implementing the action + * @data: Pointer to data passed to @action implementation + * + * Releases and removes instance of @action previously added by + * devm_add_action(). Both action and data should match one of the + * existing entries. */ -void devm_remove_action(struct device *dev, void (*action)(void *), void *data) +void devm_release_action(struct device *dev, void (*action)(void *), void *data) { struct action_devres devres = { .data = data, .action = action, }; - WARN_ON(devres_destroy(dev, devm_action_release, devm_action_match, + WARN_ON(devres_release(dev, devm_action_release, devm_action_match, &devres)); } -EXPORT_SYMBOL_GPL(devm_remove_action); +EXPORT_SYMBOL_GPL(devm_release_action); /* - * Managed kzalloc/kfree + * Managed kmalloc/kfree */ -static void devm_kzalloc_release(struct device *dev, void *res) +static void devm_kmalloc_release(struct device *dev, void *res) { /* noop */ } -static int devm_kzalloc_match(struct device *dev, void *res, void *data) +static int devm_kmalloc_match(struct device *dev, void *res, void *data) { return res == data; } /** - * devm_kzalloc - Resource-managed kzalloc + * devm_kmalloc - Resource-managed kmalloc * @dev: Device to allocate memory for * @size: Allocation size * @gfp: Allocation gfp flags * - * Managed kzalloc. Memory allocated with this function is + * Managed kmalloc. Memory allocated with this function is * automatically freed on driver detach. Like all other devres * resources, guaranteed alignment is unsigned long long. * * RETURNS: * Pointer to allocated memory on success, NULL on failure. */ -void * devm_kzalloc(struct device *dev, size_t size, gfp_t gfp) +void *devm_kmalloc(struct device *dev, size_t size, gfp_t gfp) { struct devres *dr; + if (unlikely(!size)) + return ZERO_SIZE_PTR; + /* use raw alloc_dr for kmalloc caller tracing */ - dr = alloc_dr(devm_kzalloc_release, size, gfp); + dr = alloc_dr(devm_kmalloc_release, size, gfp, dev_to_node(dev)); if (unlikely(!dr)) return NULL; + /* + * This is named devm_kzalloc_release for historical reasons + * The initial implementation did not support kmalloc, only kzalloc + */ set_node_dbginfo(&dr->node, "devm_kzalloc_release", size); devres_add(dev, dr->data); return dr->data; } -EXPORT_SYMBOL_GPL(devm_kzalloc); +EXPORT_SYMBOL_GPL(devm_kmalloc); + +/** + * devm_krealloc - Resource-managed krealloc() + * @dev: Device to re-allocate memory for + * @ptr: Pointer to the memory chunk to re-allocate + * @new_size: New allocation size + * @gfp: Allocation gfp flags + * + * Managed krealloc(). Resizes the memory chunk allocated with devm_kmalloc(). + * Behaves similarly to regular krealloc(): if @ptr is NULL or ZERO_SIZE_PTR, + * it's the equivalent of devm_kmalloc(). If new_size is zero, it frees the + * previously allocated memory and returns ZERO_SIZE_PTR. This function doesn't + * change the order in which the release callback for the re-alloc'ed devres + * will be called (except when falling back to devm_kmalloc() or when freeing + * resources when new_size is zero). The contents of the memory are preserved + * up to the lesser of new and old sizes. + */ +void *devm_krealloc(struct device *dev, void *ptr, size_t new_size, gfp_t gfp) +{ + size_t total_new_size, total_old_size; + struct devres *old_dr, *new_dr; + unsigned long flags; + + if (unlikely(!new_size)) { + devm_kfree(dev, ptr); + return ZERO_SIZE_PTR; + } + + if (unlikely(ZERO_OR_NULL_PTR(ptr))) + return devm_kmalloc(dev, new_size, gfp); + + if (WARN_ON(is_kernel_rodata((unsigned long)ptr))) + /* + * We cannot reliably realloc a const string returned by + * devm_kstrdup_const(). + */ + return NULL; + + if (!check_dr_size(new_size, &total_new_size)) + return NULL; + + total_old_size = ksize(container_of(ptr, struct devres, data)); + if (total_old_size == 0) { + WARN(1, "Pointer doesn't point to dynamically allocated memory."); + return NULL; + } + + /* + * If new size is smaller or equal to the actual number of bytes + * allocated previously - just return the same pointer. + */ + if (total_new_size <= total_old_size) + return ptr; + + /* + * Otherwise: allocate new, larger chunk. We need to allocate before + * taking the lock as most probably the caller uses GFP_KERNEL. + * alloc_dr() will call check_dr_size() to reserve extra memory + * for struct devres automatically, so size @new_size user request + * is delivered to it directly as devm_kmalloc() does. + */ + new_dr = alloc_dr(devm_kmalloc_release, + new_size, gfp, dev_to_node(dev)); + if (!new_dr) + return NULL; + + /* + * The spinlock protects the linked list against concurrent + * modifications but not the resource itself. + */ + spin_lock_irqsave(&dev->devres_lock, flags); + + old_dr = find_dr(dev, devm_kmalloc_release, devm_kmalloc_match, ptr); + if (!old_dr) { + spin_unlock_irqrestore(&dev->devres_lock, flags); + kfree(new_dr); + WARN(1, "Memory chunk not managed or managed by a different device."); + return NULL; + } + + replace_dr(dev, &old_dr->node, &new_dr->node); + + spin_unlock_irqrestore(&dev->devres_lock, flags); + + /* + * We can copy the memory contents after releasing the lock as we're + * no longer modifying the list links. + */ + memcpy(new_dr->data, old_dr->data, + total_old_size - offsetof(struct devres, data)); + /* + * Same for releasing the old devres - it's now been removed from the + * list. This is also the reason why we must not use devm_kfree() - the + * links are no longer valid. + */ + kfree(old_dr); + + return new_dr->data; +} +EXPORT_SYMBOL_GPL(devm_krealloc); + +/** + * devm_kstrdup - Allocate resource managed space and + * copy an existing string into that. + * @dev: Device to allocate memory for + * @s: the string to duplicate + * @gfp: the GFP mask used in the devm_kmalloc() call when + * allocating memory + * RETURNS: + * Pointer to allocated string on success, NULL on failure. + */ +char *devm_kstrdup(struct device *dev, const char *s, gfp_t gfp) +{ + if (!s) + return NULL; + + return devm_kmemdup(dev, s, strlen(s) + 1, gfp); +} +EXPORT_SYMBOL_GPL(devm_kstrdup); + +/** + * devm_kstrdup_const - resource managed conditional string duplication + * @dev: device for which to duplicate the string + * @s: the string to duplicate + * @gfp: the GFP mask used in the kmalloc() call when allocating memory + * + * Strings allocated by devm_kstrdup_const will be automatically freed when + * the associated device is detached. + * + * RETURNS: + * Source string if it is in .rodata section otherwise it falls back to + * devm_kstrdup. + */ +const char *devm_kstrdup_const(struct device *dev, const char *s, gfp_t gfp) +{ + if (is_kernel_rodata((unsigned long)s)) + return s; + + return devm_kstrdup(dev, s, gfp); +} +EXPORT_SYMBOL_GPL(devm_kstrdup_const); + +/** + * devm_kvasprintf - Allocate resource managed space and format a string + * into that. + * @dev: Device to allocate memory for + * @gfp: the GFP mask used in the devm_kmalloc() call when + * allocating memory + * @fmt: The printf()-style format string + * @ap: Arguments for the format string + * RETURNS: + * Pointer to allocated string on success, NULL on failure. + */ +char *devm_kvasprintf(struct device *dev, gfp_t gfp, const char *fmt, + va_list ap) +{ + unsigned int len; + char *p; + va_list aq; + + va_copy(aq, ap); + len = vsnprintf(NULL, 0, fmt, aq); + va_end(aq); + + p = devm_kmalloc(dev, len+1, gfp); + if (!p) + return NULL; + + vsnprintf(p, len+1, fmt, ap); + + return p; +} +EXPORT_SYMBOL(devm_kvasprintf); + +/** + * devm_kasprintf - Allocate resource managed space and format a string + * into that. + * @dev: Device to allocate memory for + * @gfp: the GFP mask used in the devm_kmalloc() call when + * allocating memory + * @fmt: The printf()-style format string + * @...: Arguments for the format string + * RETURNS: + * Pointer to allocated string on success, NULL on failure. + */ +char *devm_kasprintf(struct device *dev, gfp_t gfp, const char *fmt, ...) +{ + va_list ap; + char *p; + + va_start(ap, fmt); + p = devm_kvasprintf(dev, gfp, fmt, ap); + va_end(ap); + + return p; +} +EXPORT_SYMBOL_GPL(devm_kasprintf); /** * devm_kfree - Resource-managed kfree * @dev: Device this memory belongs to * @p: Memory to free * - * Free memory allocated with devm_kzalloc(). + * Free memory allocated with devm_kmalloc(). */ -void devm_kfree(struct device *dev, void *p) +void devm_kfree(struct device *dev, const void *p) { int rc; - rc = devres_destroy(dev, devm_kzalloc_release, devm_kzalloc_match, p); + /* + * Special cases: pointer to a string in .rodata returned by + * devm_kstrdup_const() or NULL/ZERO ptr. + */ + if (unlikely(is_kernel_rodata((unsigned long)p) || ZERO_OR_NULL_PTR(p))) + return; + + rc = devres_destroy(dev, devm_kmalloc_release, + devm_kmalloc_match, (void *)p); WARN_ON(rc); } EXPORT_SYMBOL_GPL(devm_kfree); + +/** + * devm_kmemdup - Resource-managed kmemdup + * @dev: Device this memory belongs to + * @src: Memory region to duplicate + * @len: Memory region length + * @gfp: GFP mask to use + * + * Duplicate region of a memory using resource managed kmalloc + */ +void *devm_kmemdup(struct device *dev, const void *src, size_t len, gfp_t gfp) +{ + void *p; + + p = devm_kmalloc(dev, len, gfp); + if (p) + memcpy(p, src, len); + + return p; +} +EXPORT_SYMBOL_GPL(devm_kmemdup); + +/** + * devm_kmemdup_const - conditionally duplicate and manage a region of memory + * + * @dev: Device this memory belongs to + * @src: memory region to duplicate + * @len: memory region length, + * @gfp: GFP mask to use + * + * Return: source address if it is in .rodata or the return value of kmemdup() + * to which the function falls back otherwise. + */ +const void * +devm_kmemdup_const(struct device *dev, const void *src, size_t len, gfp_t gfp) +{ + if (is_kernel_rodata((unsigned long)src)) + return src; + + return devm_kmemdup(dev, src, len, gfp); +} +EXPORT_SYMBOL_GPL(devm_kmemdup_const); + +struct pages_devres { + unsigned long addr; + unsigned int order; +}; + +static int devm_pages_match(struct device *dev, void *res, void *p) +{ + struct pages_devres *devres = res; + struct pages_devres *target = p; + + return devres->addr == target->addr; +} + +static void devm_pages_release(struct device *dev, void *res) +{ + struct pages_devres *devres = res; + + free_pages(devres->addr, devres->order); +} + +/** + * devm_get_free_pages - Resource-managed __get_free_pages + * @dev: Device to allocate memory for + * @gfp_mask: Allocation gfp flags + * @order: Allocation size is (1 << order) pages + * + * Managed get_free_pages. Memory allocated with this function is + * automatically freed on driver detach. + * + * RETURNS: + * Address of allocated memory on success, 0 on failure. + */ + +unsigned long devm_get_free_pages(struct device *dev, + gfp_t gfp_mask, unsigned int order) +{ + struct pages_devres *devres; + unsigned long addr; + + addr = __get_free_pages(gfp_mask, order); + + if (unlikely(!addr)) + return 0; + + devres = devres_alloc(devm_pages_release, + sizeof(struct pages_devres), GFP_KERNEL); + if (unlikely(!devres)) { + free_pages(addr, order); + return 0; + } + + devres->addr = addr; + devres->order = order; + + devres_add(dev, devres); + return addr; +} +EXPORT_SYMBOL_GPL(devm_get_free_pages); + +/** + * devm_free_pages - Resource-managed free_pages + * @dev: Device this memory belongs to + * @addr: Memory to free + * + * Free memory allocated with devm_get_free_pages(). Unlike free_pages, + * there is no need to supply the @order. + */ +void devm_free_pages(struct device *dev, unsigned long addr) +{ + struct pages_devres devres = { .addr = addr }; + + WARN_ON(devres_release(dev, devm_pages_release, devm_pages_match, + &devres)); +} +EXPORT_SYMBOL_GPL(devm_free_pages); + +static void devm_percpu_release(struct device *dev, void *pdata) +{ + void __percpu *p; + + p = *(void __percpu **)pdata; + free_percpu(p); +} + +/** + * __devm_alloc_percpu - Resource-managed alloc_percpu + * @dev: Device to allocate per-cpu memory for + * @size: Size of per-cpu memory to allocate + * @align: Alignment of per-cpu memory to allocate + * + * Managed alloc_percpu. Per-cpu memory allocated with this function is + * automatically freed on driver detach. + * + * RETURNS: + * Pointer to allocated memory on success, NULL on failure. + */ +void __percpu *__devm_alloc_percpu(struct device *dev, size_t size, + size_t align) +{ + void *p; + void __percpu *pcpu; + + pcpu = __alloc_percpu(size, align); + if (!pcpu) + return NULL; + + p = devres_alloc(devm_percpu_release, sizeof(void *), GFP_KERNEL); + if (!p) { + free_percpu(pcpu); + return NULL; + } + + *(void __percpu **)p = pcpu; + + devres_add(dev, p); + + return pcpu; +} +EXPORT_SYMBOL_GPL(__devm_alloc_percpu); diff --git a/drivers/base/devtmpfs.c b/drivers/base/devtmpfs.c index 7413d065906b..194b44075ac7 100644 --- a/drivers/base/devtmpfs.c +++ b/drivers/base/devtmpfs.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * devtmpfs - kernel-maintained tmpfs-based /dev * @@ -12,11 +13,13 @@ * overwrite the default setting if needed. */ +#define pr_fmt(fmt) "devtmpfs: " fmt + #include <linux/kernel.h> #include <linux/syscalls.h> #include <linux/mount.h> #include <linux/device.h> -#include <linux/genhd.h> +#include <linux/blkdev.h> #include <linux/namei.h> #include <linux/fs.h> #include <linux/shmem_fs.h> @@ -24,16 +27,20 @@ #include <linux/sched.h> #include <linux/slab.h> #include <linux/kthread.h> +#include <linux/init_syscalls.h> +#include <uapi/linux/mount.h> #include "base.h" -static struct task_struct *thread; - -#if defined CONFIG_DEVTMPFS_MOUNT -static int mount_dev = 1; +#ifdef CONFIG_DEVTMPFS_SAFE +#define DEVTMPFS_MFLAGS (MS_SILENT | MS_NOEXEC | MS_NOSUID) #else -static int mount_dev; +#define DEVTMPFS_MFLAGS (MS_SILENT) #endif +static struct task_struct *thread; + +static int __initdata mount_dev = IS_ENABLED(CONFIG_DEVTMPFS_MOUNT); + static DEFINE_SPINLOCK(req_lock); static struct req { @@ -54,30 +61,70 @@ static int __init mount_param(char *str) } __setup("devtmpfs.mount=", mount_param); -static struct dentry *dev_mount(struct file_system_type *fs_type, int flags, - const char *dev_name, void *data) +static struct vfsmount *mnt; + +static struct file_system_type internal_fs_type = { + .name = "devtmpfs", +#ifdef CONFIG_TMPFS + .init_fs_context = shmem_init_fs_context, +#else + .init_fs_context = ramfs_init_fs_context, +#endif + .kill_sb = kill_anon_super, +}; + +/* Simply take a ref on the existing mount */ +static int devtmpfs_get_tree(struct fs_context *fc) { + struct super_block *sb = mnt->mnt_sb; + + atomic_inc(&sb->s_active); + down_write(&sb->s_umount); + fc->root = dget(sb->s_root); + return 0; +} + +/* Ops are filled in during init depending on underlying shmem or ramfs type */ +struct fs_context_operations devtmpfs_context_ops = {}; + +/* Call the underlying initialization and set to our ops */ +static int devtmpfs_init_fs_context(struct fs_context *fc) +{ + int ret; #ifdef CONFIG_TMPFS - return mount_single(fs_type, flags, data, shmem_fill_super); + ret = shmem_init_fs_context(fc); #else - return mount_single(fs_type, flags, data, ramfs_fill_super); + ret = ramfs_init_fs_context(fc); #endif + if (ret < 0) + return ret; + + fc->ops = &devtmpfs_context_ops; + + return 0; } static struct file_system_type dev_fs_type = { .name = "devtmpfs", - .mount = dev_mount, - .kill_sb = kill_litter_super, + .init_fs_context = devtmpfs_init_fs_context, }; -#ifdef CONFIG_BLOCK -static inline int is_blockdev(struct device *dev) +static int devtmpfs_submit_req(struct req *req, const char *tmp) { - return dev->class == &block_class; + init_completion(&req->done); + + spin_lock(&req_lock); + req->next = requests; + requests = req; + spin_unlock(&req_lock); + + wake_up_process(thread); + wait_for_completion(&req->done); + + kfree(tmp); + + return req->err; } -#else -static inline int is_blockdev(struct device *dev) { return 0; } -#endif int devtmpfs_create_node(struct device *dev) { @@ -103,19 +150,7 @@ int devtmpfs_create_node(struct device *dev) req.dev = dev; - init_completion(&req.done); - - spin_lock(&req_lock); - req.next = requests; - requests = &req; - spin_unlock(&req_lock); - - wake_up_process(thread); - wait_for_completion(&req.done); - - kfree(tmp); - - return req.err; + return devtmpfs_submit_req(&req, tmp); } int devtmpfs_delete_node(struct device *dev) @@ -133,36 +168,24 @@ int devtmpfs_delete_node(struct device *dev) req.mode = 0; req.dev = dev; - init_completion(&req.done); - - spin_lock(&req_lock); - req.next = requests; - requests = &req; - spin_unlock(&req_lock); - - wake_up_process(thread); - wait_for_completion(&req.done); - - kfree(tmp); - return req.err; + return devtmpfs_submit_req(&req, tmp); } static int dev_mkdir(const char *name, umode_t mode) { struct dentry *dentry; struct path path; - int err; - dentry = kern_path_create(AT_FDCWD, name, &path, LOOKUP_DIRECTORY); + dentry = start_creating_path(AT_FDCWD, name, &path, LOOKUP_DIRECTORY); if (IS_ERR(dentry)) return PTR_ERR(dentry); - err = vfs_mkdir(path.dentry->d_inode, dentry, mode); - if (!err) + dentry = vfs_mkdir(&nop_mnt_idmap, d_inode(path.dentry), dentry, mode, NULL); + if (!IS_ERR(dentry)) /* mark as kernel-created inode */ - dentry->d_inode->i_private = &thread; - done_path_create(&path, dentry); - return err; + d_inode(dentry)->i_private = &thread; + end_creating_path(&path, dentry); + return PTR_ERR_OR_ZERO(dentry); } static int create_path(const char *nodepath) @@ -199,15 +222,16 @@ static int handle_create(const char *nodename, umode_t mode, kuid_t uid, struct path path; int err; - dentry = kern_path_create(AT_FDCWD, nodename, &path, 0); + dentry = start_creating_path(AT_FDCWD, nodename, &path, 0); if (dentry == ERR_PTR(-ENOENT)) { create_path(nodename); - dentry = kern_path_create(AT_FDCWD, nodename, &path, 0); + dentry = start_creating_path(AT_FDCWD, nodename, &path, 0); } if (IS_ERR(dentry)) return PTR_ERR(dentry); - err = vfs_mknod(path.dentry->d_inode, dentry, mode, dev->devt); + err = vfs_mknod(&nop_mnt_idmap, d_inode(path.dentry), dentry, mode, + dev->devt, NULL); if (!err) { struct iattr newattrs; @@ -215,14 +239,14 @@ static int handle_create(const char *nodename, umode_t mode, kuid_t uid, newattrs.ia_uid = uid; newattrs.ia_gid = gid; newattrs.ia_valid = ATTR_MODE|ATTR_UID|ATTR_GID; - mutex_lock(&dentry->d_inode->i_mutex); - notify_change(dentry, &newattrs); - mutex_unlock(&dentry->d_inode->i_mutex); + inode_lock(d_inode(dentry)); + notify_change(&nop_mnt_idmap, dentry, &newattrs, NULL); + inode_unlock(d_inode(dentry)); /* mark as kernel-created inode */ - dentry->d_inode->i_private = &thread; + d_inode(dentry)->i_private = &thread; } - done_path_create(&path, dentry); + end_creating_path(&path, dentry); return err; } @@ -232,26 +256,22 @@ static int dev_rmdir(const char *name) struct dentry *dentry; int err; - dentry = kern_path_locked(name, &parent); + dentry = start_removing_path(name, &parent); if (IS_ERR(dentry)) return PTR_ERR(dentry); - if (dentry->d_inode) { - if (dentry->d_inode->i_private == &thread) - err = vfs_rmdir(parent.dentry->d_inode, dentry); - else - err = -EPERM; - } else { - err = -ENOENT; - } - dput(dentry); - mutex_unlock(&parent.dentry->d_inode->i_mutex); - path_put(&parent); + if (d_inode(dentry)->i_private == &thread) + err = vfs_rmdir(&nop_mnt_idmap, d_inode(parent.dentry), + dentry, NULL); + else + err = -EPERM; + + end_removing_path(&parent, dentry); return err; } static int delete_path(const char *nodepath) { - const char *path; + char *path; int err = 0; path = kstrdup(nodepath, GFP_KERNEL); @@ -274,7 +294,7 @@ static int delete_path(const char *nodepath) return err; } -static int dev_mynode(struct device *dev, struct inode *inode, struct kstat *stat) +static int dev_mynode(struct device *dev, struct inode *inode) { /* did we create it */ if (inode->i_private != &thread) @@ -282,13 +302,13 @@ static int dev_mynode(struct device *dev, struct inode *inode, struct kstat *sta /* does the dev_t match */ if (is_blockdev(dev)) { - if (!S_ISBLK(stat->mode)) + if (!S_ISBLK(inode->i_mode)) return 0; } else { - if (!S_ISCHR(stat->mode)) + if (!S_ISCHR(inode->i_mode)) return 0; } - if (stat->rdev != dev->devt) + if (inode->i_rdev != dev->devt) return 0; /* ours */ @@ -299,42 +319,36 @@ static int handle_remove(const char *nodename, struct device *dev) { struct path parent; struct dentry *dentry; - int deleted = 1; - int err; + struct inode *inode; + int deleted = 0; + int err = 0; - dentry = kern_path_locked(nodename, &parent); + dentry = start_removing_path(nodename, &parent); if (IS_ERR(dentry)) return PTR_ERR(dentry); - if (dentry->d_inode) { - struct kstat stat; - struct path p = {.mnt = parent.mnt, .dentry = dentry}; - err = vfs_getattr(&p, &stat); - if (!err && dev_mynode(dev, dentry->d_inode, &stat)) { - struct iattr newattrs; - /* - * before unlinking this node, reset permissions - * of possible references like hardlinks - */ - newattrs.ia_uid = GLOBAL_ROOT_UID; - newattrs.ia_gid = GLOBAL_ROOT_GID; - newattrs.ia_mode = stat.mode & ~0777; - newattrs.ia_valid = - ATTR_UID|ATTR_GID|ATTR_MODE; - mutex_lock(&dentry->d_inode->i_mutex); - notify_change(dentry, &newattrs); - mutex_unlock(&dentry->d_inode->i_mutex); - err = vfs_unlink(parent.dentry->d_inode, dentry); - if (!err || err == -ENOENT) - deleted = 1; - } - } else { - err = -ENOENT; + inode = d_inode(dentry); + if (dev_mynode(dev, inode)) { + struct iattr newattrs; + /* + * before unlinking this node, reset permissions + * of possible references like hardlinks + */ + newattrs.ia_uid = GLOBAL_ROOT_UID; + newattrs.ia_gid = GLOBAL_ROOT_GID; + newattrs.ia_mode = inode->i_mode & ~0777; + newattrs.ia_valid = + ATTR_UID|ATTR_GID|ATTR_MODE; + inode_lock(d_inode(dentry)); + notify_change(&nop_mnt_idmap, dentry, &newattrs, NULL); + inode_unlock(d_inode(dentry)); + err = vfs_unlink(&nop_mnt_idmap, d_inode(parent.dentry), + dentry, NULL); + if (!err || err == -ENOENT) + deleted = 1; } - dput(dentry); - mutex_unlock(&parent.dentry->d_inode->i_mutex); + end_removing_path(&parent, dentry); - path_put(&parent); if (deleted && strchr(nodename, '/')) delete_path(nodename); return err; @@ -344,7 +358,7 @@ static int handle_remove(const char *nodename, struct device *dev) * If configured, or requested by the commandline, devtmpfs will be * auto-mounted after the kernel mounted the root filesystem. */ -int devtmpfs_mount(const char *mntdir) +int __init devtmpfs_mount(void) { int err; @@ -354,15 +368,15 @@ int devtmpfs_mount(const char *mntdir) if (!thread) return 0; - err = sys_mount("devtmpfs", (char *)mntdir, "devtmpfs", MS_SILENT, NULL); + err = init_mount("devtmpfs", "dev", "devtmpfs", DEVTMPFS_MFLAGS, NULL); if (err) - printk(KERN_INFO "devtmpfs: error mounting %i\n", err); + pr_info("error mounting %d\n", err); else - printk(KERN_INFO "devtmpfs: mounted\n"); + pr_info("mounted\n"); return err; } -static DECLARE_COMPLETION(setup_done); +static __initdata DECLARE_COMPLETION(setup_done); static int handle(const char *name, umode_t mode, kuid_t uid, kgid_t gid, struct device *dev) @@ -373,19 +387,8 @@ static int handle(const char *name, umode_t mode, kuid_t uid, kgid_t gid, return handle_remove(name, dev); } -static int devtmpfsd(void *p) +static void __noreturn devtmpfs_work_loop(void) { - char options[] = "mode=0755"; - int *err = p; - *err = sys_unshare(CLONE_NEWNS); - if (*err) - goto out; - *err = sys_mount("devtmpfs", "/", "devtmpfs", MS_SILENT, options); - if (*err) - goto out; - sys_chdir("/.."); /* will traverse into overmounted root */ - sys_chroot("."); - complete(&setup_done); while (1) { spin_lock(&req_lock); while (requests) { @@ -405,10 +408,64 @@ static int devtmpfsd(void *p) spin_unlock(&req_lock); schedule(); } - return 0; +} + +static noinline int __init devtmpfs_setup(void *p) +{ + int err; + + err = ksys_unshare(CLONE_NEWNS); + if (err) + goto out; + err = init_mount("devtmpfs", "/", "devtmpfs", DEVTMPFS_MFLAGS, NULL); + if (err) + goto out; + init_chdir("/.."); /* will traverse into overmounted root */ + init_chroot("."); out: + *(int *)p = err; + return err; +} + +/* + * The __ref is because devtmpfs_setup needs to be __init for the routines it + * calls. That call is done while devtmpfs_init, which is marked __init, + * synchronously waits for it to complete. + */ +static int __ref devtmpfsd(void *p) +{ + int err = devtmpfs_setup(p); + complete(&setup_done); - return *err; + if (err) + return err; + devtmpfs_work_loop(); + return 0; +} + +/* + * Get the underlying (shmem/ramfs) context ops to build ours + */ +static int devtmpfs_configure_context(void) +{ + struct fs_context *fc; + + fc = fs_context_for_reconfigure(mnt->mnt_root, mnt->mnt_sb->s_flags, + MS_RMT_MASK); + if (IS_ERR(fc)) + return PTR_ERR(fc); + + /* Set up devtmpfs_context_ops based on underlying type */ + devtmpfs_context_ops.free = fc->ops->free; + devtmpfs_context_ops.dup = fc->ops->dup; + devtmpfs_context_ops.parse_param = fc->ops->parse_param; + devtmpfs_context_ops.parse_monolithic = fc->ops->parse_monolithic; + devtmpfs_context_ops.get_tree = &devtmpfs_get_tree; + devtmpfs_context_ops.reconfigure = fc->ops->reconfigure; + + put_fs_context(fc); + + return 0; } /* @@ -417,10 +474,24 @@ out: */ int __init devtmpfs_init(void) { - int err = register_filesystem(&dev_fs_type); + char opts[] = "mode=0755"; + int err; + + mnt = vfs_kern_mount(&internal_fs_type, 0, "devtmpfs", opts); + if (IS_ERR(mnt)) { + pr_err("unable to create devtmpfs %ld\n", PTR_ERR(mnt)); + return PTR_ERR(mnt); + } + + err = devtmpfs_configure_context(); + if (err) { + pr_err("unable to configure devtmpfs type %d\n", err); + return err; + } + + err = register_filesystem(&dev_fs_type); if (err) { - printk(KERN_ERR "devtmpfs: unable to register devtmpfs " - "type %i\n", err); + pr_err("unable to register devtmpfs type %d\n", err); return err; } @@ -433,11 +504,12 @@ int __init devtmpfs_init(void) } if (err) { - printk(KERN_ERR "devtmpfs: unable to create devtmpfs %i\n", err); + pr_err("unable to create devtmpfs %d\n", err); unregister_filesystem(&dev_fs_type); + thread = NULL; return err; } - printk(KERN_INFO "devtmpfs: initialized\n"); + pr_info("initialized\n"); return 0; } diff --git a/drivers/base/dma-buf.c b/drivers/base/dma-buf.c deleted file mode 100644 index 6687ba741879..000000000000 --- a/drivers/base/dma-buf.c +++ /dev/null @@ -1,708 +0,0 @@ -/* - * Framework for buffer objects that can be shared across devices/subsystems. - * - * Copyright(C) 2011 Linaro Limited. All rights reserved. - * Author: Sumit Semwal <sumit.semwal@ti.com> - * - * Many thanks to linaro-mm-sig list, and specially - * Arnd Bergmann <arnd@arndb.de>, Rob Clark <rob@ti.com> and - * Daniel Vetter <daniel@ffwll.ch> for their support in creation and - * refining of this idea. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 as published by - * the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * You should have received a copy of the GNU General Public License along with - * this program. If not, see <http://www.gnu.org/licenses/>. - */ - -#include <linux/fs.h> -#include <linux/slab.h> -#include <linux/dma-buf.h> -#include <linux/anon_inodes.h> -#include <linux/export.h> -#include <linux/debugfs.h> -#include <linux/seq_file.h> - -static inline int is_dma_buf_file(struct file *); - -struct dma_buf_list { - struct list_head head; - struct mutex lock; -}; - -static struct dma_buf_list db_list; - -static int dma_buf_release(struct inode *inode, struct file *file) -{ - struct dma_buf *dmabuf; - - if (!is_dma_buf_file(file)) - return -EINVAL; - - dmabuf = file->private_data; - - BUG_ON(dmabuf->vmapping_counter); - - dmabuf->ops->release(dmabuf); - - mutex_lock(&db_list.lock); - list_del(&dmabuf->list_node); - mutex_unlock(&db_list.lock); - - kfree(dmabuf); - return 0; -} - -static int dma_buf_mmap_internal(struct file *file, struct vm_area_struct *vma) -{ - struct dma_buf *dmabuf; - - if (!is_dma_buf_file(file)) - return -EINVAL; - - dmabuf = file->private_data; - - /* check for overflowing the buffer's size */ - if (vma->vm_pgoff + ((vma->vm_end - vma->vm_start) >> PAGE_SHIFT) > - dmabuf->size >> PAGE_SHIFT) - return -EINVAL; - - return dmabuf->ops->mmap(dmabuf, vma); -} - -static const struct file_operations dma_buf_fops = { - .release = dma_buf_release, - .mmap = dma_buf_mmap_internal, -}; - -/* - * is_dma_buf_file - Check if struct file* is associated with dma_buf - */ -static inline int is_dma_buf_file(struct file *file) -{ - return file->f_op == &dma_buf_fops; -} - -/** - * dma_buf_export_named - Creates a new dma_buf, and associates an anon file - * with this buffer, so it can be exported. - * Also connect the allocator specific data and ops to the buffer. - * Additionally, provide a name string for exporter; useful in debugging. - * - * @priv: [in] Attach private data of allocator to this buffer - * @ops: [in] Attach allocator-defined dma buf ops to the new buffer. - * @size: [in] Size of the buffer - * @flags: [in] mode flags for the file. - * @exp_name: [in] name of the exporting module - useful for debugging. - * - * Returns, on success, a newly created dma_buf object, which wraps the - * supplied private data and operations for dma_buf_ops. On either missing - * ops, or error in allocating struct dma_buf, will return negative error. - * - */ -struct dma_buf *dma_buf_export_named(void *priv, const struct dma_buf_ops *ops, - size_t size, int flags, const char *exp_name) -{ - struct dma_buf *dmabuf; - struct file *file; - - if (WARN_ON(!priv || !ops - || !ops->map_dma_buf - || !ops->unmap_dma_buf - || !ops->release - || !ops->kmap_atomic - || !ops->kmap - || !ops->mmap)) { - return ERR_PTR(-EINVAL); - } - - dmabuf = kzalloc(sizeof(struct dma_buf), GFP_KERNEL); - if (dmabuf == NULL) - return ERR_PTR(-ENOMEM); - - dmabuf->priv = priv; - dmabuf->ops = ops; - dmabuf->size = size; - dmabuf->exp_name = exp_name; - - file = anon_inode_getfile("dmabuf", &dma_buf_fops, dmabuf, flags); - - dmabuf->file = file; - - mutex_init(&dmabuf->lock); - INIT_LIST_HEAD(&dmabuf->attachments); - - mutex_lock(&db_list.lock); - list_add(&dmabuf->list_node, &db_list.head); - mutex_unlock(&db_list.lock); - - return dmabuf; -} -EXPORT_SYMBOL_GPL(dma_buf_export_named); - - -/** - * dma_buf_fd - returns a file descriptor for the given dma_buf - * @dmabuf: [in] pointer to dma_buf for which fd is required. - * @flags: [in] flags to give to fd - * - * On success, returns an associated 'fd'. Else, returns error. - */ -int dma_buf_fd(struct dma_buf *dmabuf, int flags) -{ - int fd; - - if (!dmabuf || !dmabuf->file) - return -EINVAL; - - fd = get_unused_fd_flags(flags); - if (fd < 0) - return fd; - - fd_install(fd, dmabuf->file); - - return fd; -} -EXPORT_SYMBOL_GPL(dma_buf_fd); - -/** - * dma_buf_get - returns the dma_buf structure related to an fd - * @fd: [in] fd associated with the dma_buf to be returned - * - * On success, returns the dma_buf structure associated with an fd; uses - * file's refcounting done by fget to increase refcount. returns ERR_PTR - * otherwise. - */ -struct dma_buf *dma_buf_get(int fd) -{ - struct file *file; - - file = fget(fd); - - if (!file) - return ERR_PTR(-EBADF); - - if (!is_dma_buf_file(file)) { - fput(file); - return ERR_PTR(-EINVAL); - } - - return file->private_data; -} -EXPORT_SYMBOL_GPL(dma_buf_get); - -/** - * dma_buf_put - decreases refcount of the buffer - * @dmabuf: [in] buffer to reduce refcount of - * - * Uses file's refcounting done implicitly by fput() - */ -void dma_buf_put(struct dma_buf *dmabuf) -{ - if (WARN_ON(!dmabuf || !dmabuf->file)) - return; - - fput(dmabuf->file); -} -EXPORT_SYMBOL_GPL(dma_buf_put); - -/** - * dma_buf_attach - Add the device to dma_buf's attachments list; optionally, - * calls attach() of dma_buf_ops to allow device-specific attach functionality - * @dmabuf: [in] buffer to attach device to. - * @dev: [in] device to be attached. - * - * Returns struct dma_buf_attachment * for this attachment; may return negative - * error codes. - * - */ -struct dma_buf_attachment *dma_buf_attach(struct dma_buf *dmabuf, - struct device *dev) -{ - struct dma_buf_attachment *attach; - int ret; - - if (WARN_ON(!dmabuf || !dev)) - return ERR_PTR(-EINVAL); - - attach = kzalloc(sizeof(struct dma_buf_attachment), GFP_KERNEL); - if (attach == NULL) - return ERR_PTR(-ENOMEM); - - attach->dev = dev; - attach->dmabuf = dmabuf; - - mutex_lock(&dmabuf->lock); - - if (dmabuf->ops->attach) { - ret = dmabuf->ops->attach(dmabuf, dev, attach); - if (ret) - goto err_attach; - } - list_add(&attach->node, &dmabuf->attachments); - - mutex_unlock(&dmabuf->lock); - return attach; - -err_attach: - kfree(attach); - mutex_unlock(&dmabuf->lock); - return ERR_PTR(ret); -} -EXPORT_SYMBOL_GPL(dma_buf_attach); - -/** - * dma_buf_detach - Remove the given attachment from dmabuf's attachments list; - * optionally calls detach() of dma_buf_ops for device-specific detach - * @dmabuf: [in] buffer to detach from. - * @attach: [in] attachment to be detached; is free'd after this call. - * - */ -void dma_buf_detach(struct dma_buf *dmabuf, struct dma_buf_attachment *attach) -{ - if (WARN_ON(!dmabuf || !attach)) - return; - - mutex_lock(&dmabuf->lock); - list_del(&attach->node); - if (dmabuf->ops->detach) - dmabuf->ops->detach(dmabuf, attach); - - mutex_unlock(&dmabuf->lock); - kfree(attach); -} -EXPORT_SYMBOL_GPL(dma_buf_detach); - -/** - * dma_buf_map_attachment - Returns the scatterlist table of the attachment; - * mapped into _device_ address space. Is a wrapper for map_dma_buf() of the - * dma_buf_ops. - * @attach: [in] attachment whose scatterlist is to be returned - * @direction: [in] direction of DMA transfer - * - * Returns sg_table containing the scatterlist to be returned; may return NULL - * or ERR_PTR. - * - */ -struct sg_table *dma_buf_map_attachment(struct dma_buf_attachment *attach, - enum dma_data_direction direction) -{ - struct sg_table *sg_table = ERR_PTR(-EINVAL); - - might_sleep(); - - if (WARN_ON(!attach || !attach->dmabuf)) - return ERR_PTR(-EINVAL); - - sg_table = attach->dmabuf->ops->map_dma_buf(attach, direction); - - return sg_table; -} -EXPORT_SYMBOL_GPL(dma_buf_map_attachment); - -/** - * dma_buf_unmap_attachment - unmaps and decreases usecount of the buffer;might - * deallocate the scatterlist associated. Is a wrapper for unmap_dma_buf() of - * dma_buf_ops. - * @attach: [in] attachment to unmap buffer from - * @sg_table: [in] scatterlist info of the buffer to unmap - * @direction: [in] direction of DMA transfer - * - */ -void dma_buf_unmap_attachment(struct dma_buf_attachment *attach, - struct sg_table *sg_table, - enum dma_data_direction direction) -{ - might_sleep(); - - if (WARN_ON(!attach || !attach->dmabuf || !sg_table)) - return; - - attach->dmabuf->ops->unmap_dma_buf(attach, sg_table, - direction); -} -EXPORT_SYMBOL_GPL(dma_buf_unmap_attachment); - - -/** - * dma_buf_begin_cpu_access - Must be called before accessing a dma_buf from the - * cpu in the kernel context. Calls begin_cpu_access to allow exporter-specific - * preparations. Coherency is only guaranteed in the specified range for the - * specified access direction. - * @dmabuf: [in] buffer to prepare cpu access for. - * @start: [in] start of range for cpu access. - * @len: [in] length of range for cpu access. - * @direction: [in] length of range for cpu access. - * - * Can return negative error values, returns 0 on success. - */ -int dma_buf_begin_cpu_access(struct dma_buf *dmabuf, size_t start, size_t len, - enum dma_data_direction direction) -{ - int ret = 0; - - if (WARN_ON(!dmabuf)) - return -EINVAL; - - if (dmabuf->ops->begin_cpu_access) - ret = dmabuf->ops->begin_cpu_access(dmabuf, start, len, direction); - - return ret; -} -EXPORT_SYMBOL_GPL(dma_buf_begin_cpu_access); - -/** - * dma_buf_end_cpu_access - Must be called after accessing a dma_buf from the - * cpu in the kernel context. Calls end_cpu_access to allow exporter-specific - * actions. Coherency is only guaranteed in the specified range for the - * specified access direction. - * @dmabuf: [in] buffer to complete cpu access for. - * @start: [in] start of range for cpu access. - * @len: [in] length of range for cpu access. - * @direction: [in] length of range for cpu access. - * - * This call must always succeed. - */ -void dma_buf_end_cpu_access(struct dma_buf *dmabuf, size_t start, size_t len, - enum dma_data_direction direction) -{ - WARN_ON(!dmabuf); - - if (dmabuf->ops->end_cpu_access) - dmabuf->ops->end_cpu_access(dmabuf, start, len, direction); -} -EXPORT_SYMBOL_GPL(dma_buf_end_cpu_access); - -/** - * dma_buf_kmap_atomic - Map a page of the buffer object into kernel address - * space. The same restrictions as for kmap_atomic and friends apply. - * @dmabuf: [in] buffer to map page from. - * @page_num: [in] page in PAGE_SIZE units to map. - * - * This call must always succeed, any necessary preparations that might fail - * need to be done in begin_cpu_access. - */ -void *dma_buf_kmap_atomic(struct dma_buf *dmabuf, unsigned long page_num) -{ - WARN_ON(!dmabuf); - - return dmabuf->ops->kmap_atomic(dmabuf, page_num); -} -EXPORT_SYMBOL_GPL(dma_buf_kmap_atomic); - -/** - * dma_buf_kunmap_atomic - Unmap a page obtained by dma_buf_kmap_atomic. - * @dmabuf: [in] buffer to unmap page from. - * @page_num: [in] page in PAGE_SIZE units to unmap. - * @vaddr: [in] kernel space pointer obtained from dma_buf_kmap_atomic. - * - * This call must always succeed. - */ -void dma_buf_kunmap_atomic(struct dma_buf *dmabuf, unsigned long page_num, - void *vaddr) -{ - WARN_ON(!dmabuf); - - if (dmabuf->ops->kunmap_atomic) - dmabuf->ops->kunmap_atomic(dmabuf, page_num, vaddr); -} -EXPORT_SYMBOL_GPL(dma_buf_kunmap_atomic); - -/** - * dma_buf_kmap - Map a page of the buffer object into kernel address space. The - * same restrictions as for kmap and friends apply. - * @dmabuf: [in] buffer to map page from. - * @page_num: [in] page in PAGE_SIZE units to map. - * - * This call must always succeed, any necessary preparations that might fail - * need to be done in begin_cpu_access. - */ -void *dma_buf_kmap(struct dma_buf *dmabuf, unsigned long page_num) -{ - WARN_ON(!dmabuf); - - return dmabuf->ops->kmap(dmabuf, page_num); -} -EXPORT_SYMBOL_GPL(dma_buf_kmap); - -/** - * dma_buf_kunmap - Unmap a page obtained by dma_buf_kmap. - * @dmabuf: [in] buffer to unmap page from. - * @page_num: [in] page in PAGE_SIZE units to unmap. - * @vaddr: [in] kernel space pointer obtained from dma_buf_kmap. - * - * This call must always succeed. - */ -void dma_buf_kunmap(struct dma_buf *dmabuf, unsigned long page_num, - void *vaddr) -{ - WARN_ON(!dmabuf); - - if (dmabuf->ops->kunmap) - dmabuf->ops->kunmap(dmabuf, page_num, vaddr); -} -EXPORT_SYMBOL_GPL(dma_buf_kunmap); - - -/** - * dma_buf_mmap - Setup up a userspace mmap with the given vma - * @dmabuf: [in] buffer that should back the vma - * @vma: [in] vma for the mmap - * @pgoff: [in] offset in pages where this mmap should start within the - * dma-buf buffer. - * - * This function adjusts the passed in vma so that it points at the file of the - * dma_buf operation. It alsog adjusts the starting pgoff and does bounds - * checking on the size of the vma. Then it calls the exporters mmap function to - * set up the mapping. - * - * Can return negative error values, returns 0 on success. - */ -int dma_buf_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma, - unsigned long pgoff) -{ - struct file *oldfile; - int ret; - - if (WARN_ON(!dmabuf || !vma)) - return -EINVAL; - - /* check for offset overflow */ - if (pgoff + ((vma->vm_end - vma->vm_start) >> PAGE_SHIFT) < pgoff) - return -EOVERFLOW; - - /* check for overflowing the buffer's size */ - if (pgoff + ((vma->vm_end - vma->vm_start) >> PAGE_SHIFT) > - dmabuf->size >> PAGE_SHIFT) - return -EINVAL; - - /* readjust the vma */ - get_file(dmabuf->file); - oldfile = vma->vm_file; - vma->vm_file = dmabuf->file; - vma->vm_pgoff = pgoff; - - ret = dmabuf->ops->mmap(dmabuf, vma); - if (ret) { - /* restore old parameters on failure */ - vma->vm_file = oldfile; - fput(dmabuf->file); - } else { - if (oldfile) - fput(oldfile); - } - return ret; - -} -EXPORT_SYMBOL_GPL(dma_buf_mmap); - -/** - * dma_buf_vmap - Create virtual mapping for the buffer object into kernel - * address space. Same restrictions as for vmap and friends apply. - * @dmabuf: [in] buffer to vmap - * - * This call may fail due to lack of virtual mapping address space. - * These calls are optional in drivers. The intended use for them - * is for mapping objects linear in kernel space for high use objects. - * Please attempt to use kmap/kunmap before thinking about these interfaces. - */ -void *dma_buf_vmap(struct dma_buf *dmabuf) -{ - void *ptr; - - if (WARN_ON(!dmabuf)) - return NULL; - - if (!dmabuf->ops->vmap) - return NULL; - - mutex_lock(&dmabuf->lock); - if (dmabuf->vmapping_counter) { - dmabuf->vmapping_counter++; - BUG_ON(!dmabuf->vmap_ptr); - ptr = dmabuf->vmap_ptr; - goto out_unlock; - } - - BUG_ON(dmabuf->vmap_ptr); - - ptr = dmabuf->ops->vmap(dmabuf); - if (IS_ERR_OR_NULL(ptr)) - goto out_unlock; - - dmabuf->vmap_ptr = ptr; - dmabuf->vmapping_counter = 1; - -out_unlock: - mutex_unlock(&dmabuf->lock); - return ptr; -} -EXPORT_SYMBOL_GPL(dma_buf_vmap); - -/** - * dma_buf_vunmap - Unmap a vmap obtained by dma_buf_vmap. - * @dmabuf: [in] buffer to vunmap - * @vaddr: [in] vmap to vunmap - */ -void dma_buf_vunmap(struct dma_buf *dmabuf, void *vaddr) -{ - if (WARN_ON(!dmabuf)) - return; - - BUG_ON(!dmabuf->vmap_ptr); - BUG_ON(dmabuf->vmapping_counter == 0); - BUG_ON(dmabuf->vmap_ptr != vaddr); - - mutex_lock(&dmabuf->lock); - if (--dmabuf->vmapping_counter == 0) { - if (dmabuf->ops->vunmap) - dmabuf->ops->vunmap(dmabuf, vaddr); - dmabuf->vmap_ptr = NULL; - } - mutex_unlock(&dmabuf->lock); -} -EXPORT_SYMBOL_GPL(dma_buf_vunmap); - -#ifdef CONFIG_DEBUG_FS -static int dma_buf_describe(struct seq_file *s) -{ - int ret; - struct dma_buf *buf_obj; - struct dma_buf_attachment *attach_obj; - int count = 0, attach_count; - size_t size = 0; - - ret = mutex_lock_interruptible(&db_list.lock); - - if (ret) - return ret; - - seq_printf(s, "\nDma-buf Objects:\n"); - seq_printf(s, "\texp_name\tsize\tflags\tmode\tcount\n"); - - list_for_each_entry(buf_obj, &db_list.head, list_node) { - ret = mutex_lock_interruptible(&buf_obj->lock); - - if (ret) { - seq_printf(s, - "\tERROR locking buffer object: skipping\n"); - continue; - } - - seq_printf(s, "\t"); - - seq_printf(s, "\t%s\t%08zu\t%08x\t%08x\t%08ld\n", - buf_obj->exp_name, buf_obj->size, - buf_obj->file->f_flags, buf_obj->file->f_mode, - (long)(buf_obj->file->f_count.counter)); - - seq_printf(s, "\t\tAttached Devices:\n"); - attach_count = 0; - - list_for_each_entry(attach_obj, &buf_obj->attachments, node) { - seq_printf(s, "\t\t"); - - seq_printf(s, "%s\n", attach_obj->dev->init_name); - attach_count++; - } - - seq_printf(s, "\n\t\tTotal %d devices attached\n", - attach_count); - - count++; - size += buf_obj->size; - mutex_unlock(&buf_obj->lock); - } - - seq_printf(s, "\nTotal %d objects, %zu bytes\n", count, size); - - mutex_unlock(&db_list.lock); - return 0; -} - -static int dma_buf_show(struct seq_file *s, void *unused) -{ - void (*func)(struct seq_file *) = s->private; - func(s); - return 0; -} - -static int dma_buf_debug_open(struct inode *inode, struct file *file) -{ - return single_open(file, dma_buf_show, inode->i_private); -} - -static const struct file_operations dma_buf_debug_fops = { - .open = dma_buf_debug_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; - -static struct dentry *dma_buf_debugfs_dir; - -static int dma_buf_init_debugfs(void) -{ - int err = 0; - dma_buf_debugfs_dir = debugfs_create_dir("dma_buf", NULL); - if (IS_ERR(dma_buf_debugfs_dir)) { - err = PTR_ERR(dma_buf_debugfs_dir); - dma_buf_debugfs_dir = NULL; - return err; - } - - err = dma_buf_debugfs_create_file("bufinfo", dma_buf_describe); - - if (err) - pr_debug("dma_buf: debugfs: failed to create node bufinfo\n"); - - return err; -} - -static void dma_buf_uninit_debugfs(void) -{ - if (dma_buf_debugfs_dir) - debugfs_remove_recursive(dma_buf_debugfs_dir); -} - -int dma_buf_debugfs_create_file(const char *name, - int (*write)(struct seq_file *)) -{ - struct dentry *d; - - d = debugfs_create_file(name, S_IRUGO, dma_buf_debugfs_dir, - write, &dma_buf_debug_fops); - - return PTR_RET(d); -} -#else -static inline int dma_buf_init_debugfs(void) -{ - return 0; -} -static inline void dma_buf_uninit_debugfs(void) -{ -} -#endif - -static int __init dma_buf_init(void) -{ - mutex_init(&db_list.lock); - INIT_LIST_HEAD(&db_list.head); - dma_buf_init_debugfs(); - return 0; -} -subsys_initcall(dma_buf_init); - -static void __exit dma_buf_deinit(void) -{ - dma_buf_uninit_debugfs(); -} -__exitcall(dma_buf_deinit); diff --git a/drivers/base/dma-coherent.c b/drivers/base/dma-coherent.c deleted file mode 100644 index bc256b641027..000000000000 --- a/drivers/base/dma-coherent.c +++ /dev/null @@ -1,220 +0,0 @@ -/* - * Coherent per-device memory handling. - * Borrowed from i386 - */ -#include <linux/slab.h> -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/dma-mapping.h> - -struct dma_coherent_mem { - void *virt_base; - dma_addr_t device_base; - phys_addr_t pfn_base; - int size; - int flags; - unsigned long *bitmap; -}; - -int dma_declare_coherent_memory(struct device *dev, dma_addr_t bus_addr, - dma_addr_t device_addr, size_t size, int flags) -{ - void __iomem *mem_base = NULL; - int pages = size >> PAGE_SHIFT; - int bitmap_size = BITS_TO_LONGS(pages) * sizeof(long); - - if ((flags & (DMA_MEMORY_MAP | DMA_MEMORY_IO)) == 0) - goto out; - if (!size) - goto out; - if (dev->dma_mem) - goto out; - - /* FIXME: this routine just ignores DMA_MEMORY_INCLUDES_CHILDREN */ - - mem_base = ioremap(bus_addr, size); - if (!mem_base) - goto out; - - dev->dma_mem = kzalloc(sizeof(struct dma_coherent_mem), GFP_KERNEL); - if (!dev->dma_mem) - goto out; - dev->dma_mem->bitmap = kzalloc(bitmap_size, GFP_KERNEL); - if (!dev->dma_mem->bitmap) - goto free1_out; - - dev->dma_mem->virt_base = mem_base; - dev->dma_mem->device_base = device_addr; - dev->dma_mem->pfn_base = PFN_DOWN(bus_addr); - dev->dma_mem->size = pages; - dev->dma_mem->flags = flags; - - if (flags & DMA_MEMORY_MAP) - return DMA_MEMORY_MAP; - - return DMA_MEMORY_IO; - - free1_out: - kfree(dev->dma_mem); - out: - if (mem_base) - iounmap(mem_base); - return 0; -} -EXPORT_SYMBOL(dma_declare_coherent_memory); - -void dma_release_declared_memory(struct device *dev) -{ - struct dma_coherent_mem *mem = dev->dma_mem; - - if (!mem) - return; - dev->dma_mem = NULL; - iounmap(mem->virt_base); - kfree(mem->bitmap); - kfree(mem); -} -EXPORT_SYMBOL(dma_release_declared_memory); - -void *dma_mark_declared_memory_occupied(struct device *dev, - dma_addr_t device_addr, size_t size) -{ - struct dma_coherent_mem *mem = dev->dma_mem; - int pos, err; - - size += device_addr & ~PAGE_MASK; - - if (!mem) - return ERR_PTR(-EINVAL); - - pos = (device_addr - mem->device_base) >> PAGE_SHIFT; - err = bitmap_allocate_region(mem->bitmap, pos, get_order(size)); - if (err != 0) - return ERR_PTR(err); - return mem->virt_base + (pos << PAGE_SHIFT); -} -EXPORT_SYMBOL(dma_mark_declared_memory_occupied); - -/** - * dma_alloc_from_coherent() - try to allocate memory from the per-device coherent area - * - * @dev: device from which we allocate memory - * @size: size of requested memory area - * @dma_handle: This will be filled with the correct dma handle - * @ret: This pointer will be filled with the virtual address - * to allocated area. - * - * This function should be only called from per-arch dma_alloc_coherent() - * to support allocation from per-device coherent memory pools. - * - * Returns 0 if dma_alloc_coherent should continue with allocating from - * generic memory areas, or !0 if dma_alloc_coherent should return @ret. - */ -int dma_alloc_from_coherent(struct device *dev, ssize_t size, - dma_addr_t *dma_handle, void **ret) -{ - struct dma_coherent_mem *mem; - int order = get_order(size); - int pageno; - - if (!dev) - return 0; - mem = dev->dma_mem; - if (!mem) - return 0; - - *ret = NULL; - - if (unlikely(size > (mem->size << PAGE_SHIFT))) - goto err; - - pageno = bitmap_find_free_region(mem->bitmap, mem->size, order); - if (unlikely(pageno < 0)) - goto err; - - /* - * Memory was found in the per-device area. - */ - *dma_handle = mem->device_base + (pageno << PAGE_SHIFT); - *ret = mem->virt_base + (pageno << PAGE_SHIFT); - memset(*ret, 0, size); - - return 1; - -err: - /* - * In the case where the allocation can not be satisfied from the - * per-device area, try to fall back to generic memory if the - * constraints allow it. - */ - return mem->flags & DMA_MEMORY_EXCLUSIVE; -} -EXPORT_SYMBOL(dma_alloc_from_coherent); - -/** - * dma_release_from_coherent() - try to free the memory allocated from per-device coherent memory pool - * @dev: device from which the memory was allocated - * @order: the order of pages allocated - * @vaddr: virtual address of allocated pages - * - * This checks whether the memory was allocated from the per-device - * coherent memory pool and if so, releases that memory. - * - * Returns 1 if we correctly released the memory, or 0 if - * dma_release_coherent() should proceed with releasing memory from - * generic pools. - */ -int dma_release_from_coherent(struct device *dev, int order, void *vaddr) -{ - struct dma_coherent_mem *mem = dev ? dev->dma_mem : NULL; - - if (mem && vaddr >= mem->virt_base && vaddr < - (mem->virt_base + (mem->size << PAGE_SHIFT))) { - int page = (vaddr - mem->virt_base) >> PAGE_SHIFT; - - bitmap_release_region(mem->bitmap, page, order); - return 1; - } - return 0; -} -EXPORT_SYMBOL(dma_release_from_coherent); - -/** - * dma_mmap_from_coherent() - try to mmap the memory allocated from - * per-device coherent memory pool to userspace - * @dev: device from which the memory was allocated - * @vma: vm_area for the userspace memory - * @vaddr: cpu address returned by dma_alloc_from_coherent - * @size: size of the memory buffer allocated by dma_alloc_from_coherent - * @ret: result from remap_pfn_range() - * - * This checks whether the memory was allocated from the per-device - * coherent memory pool and if so, maps that memory to the provided vma. - * - * Returns 1 if we correctly mapped the memory, or 0 if the caller should - * proceed with mapping memory from generic pools. - */ -int dma_mmap_from_coherent(struct device *dev, struct vm_area_struct *vma, - void *vaddr, size_t size, int *ret) -{ - struct dma_coherent_mem *mem = dev ? dev->dma_mem : NULL; - - if (mem && vaddr >= mem->virt_base && vaddr + size <= - (mem->virt_base + (mem->size << PAGE_SHIFT))) { - unsigned long off = vma->vm_pgoff; - int start = (vaddr - mem->virt_base) >> PAGE_SHIFT; - int user_count = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; - int count = size >> PAGE_SHIFT; - - *ret = -ENXIO; - if (off < count && user_count <= count - off) { - unsigned pfn = mem->pfn_base + start + off; - *ret = remap_pfn_range(vma, vma->vm_start, pfn, - user_count << PAGE_SHIFT, - vma->vm_page_prot); - } - return 1; - } - return 0; -} -EXPORT_SYMBOL(dma_mmap_from_coherent); diff --git a/drivers/base/dma-contiguous.c b/drivers/base/dma-contiguous.c deleted file mode 100644 index 0ca54421ce97..000000000000 --- a/drivers/base/dma-contiguous.c +++ /dev/null @@ -1,390 +0,0 @@ -/* - * Contiguous Memory Allocator for DMA mapping framework - * Copyright (c) 2010-2011 by Samsung Electronics. - * Written by: - * Marek Szyprowski <m.szyprowski@samsung.com> - * Michal Nazarewicz <mina86@mina86.com> - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation; either version 2 of the - * License or (at your optional) any later version of the license. - */ - -#define pr_fmt(fmt) "cma: " fmt - -#ifdef CONFIG_CMA_DEBUG -#ifndef DEBUG -# define DEBUG -#endif -#endif - -#include <asm/page.h> -#include <asm/dma-contiguous.h> - -#include <linux/memblock.h> -#include <linux/err.h> -#include <linux/mm.h> -#include <linux/mutex.h> -#include <linux/page-isolation.h> -#include <linux/sizes.h> -#include <linux/slab.h> -#include <linux/swap.h> -#include <linux/mm_types.h> -#include <linux/dma-contiguous.h> - -struct cma { - unsigned long base_pfn; - unsigned long count; - unsigned long *bitmap; -}; - -struct cma *dma_contiguous_default_area; - -#ifdef CONFIG_CMA_SIZE_MBYTES -#define CMA_SIZE_MBYTES CONFIG_CMA_SIZE_MBYTES -#else -#define CMA_SIZE_MBYTES 0 -#endif - -/* - * Default global CMA area size can be defined in kernel's .config. - * This is usefull mainly for distro maintainers to create a kernel - * that works correctly for most supported systems. - * The size can be set in bytes or as a percentage of the total memory - * in the system. - * - * Users, who want to set the size of global CMA area for their system - * should use cma= kernel parameter. - */ -static const phys_addr_t size_bytes = CMA_SIZE_MBYTES * SZ_1M; -static phys_addr_t size_cmdline = -1; - -static int __init early_cma(char *p) -{ - pr_debug("%s(%s)\n", __func__, p); - size_cmdline = memparse(p, &p); - return 0; -} -early_param("cma", early_cma); - -#ifdef CONFIG_CMA_SIZE_PERCENTAGE - -static phys_addr_t __init __maybe_unused cma_early_percent_memory(void) -{ - struct memblock_region *reg; - unsigned long total_pages = 0; - - /* - * We cannot use memblock_phys_mem_size() here, because - * memblock_analyze() has not been called yet. - */ - for_each_memblock(memory, reg) - total_pages += memblock_region_memory_end_pfn(reg) - - memblock_region_memory_base_pfn(reg); - - return (total_pages * CONFIG_CMA_SIZE_PERCENTAGE / 100) << PAGE_SHIFT; -} - -#else - -static inline __maybe_unused phys_addr_t cma_early_percent_memory(void) -{ - return 0; -} - -#endif - -/** - * dma_contiguous_reserve() - reserve area for contiguous memory handling - * @limit: End address of the reserved memory (optional, 0 for any). - * - * This function reserves memory from early allocator. It should be - * called by arch specific code once the early allocator (memblock or bootmem) - * has been activated and all other subsystems have already allocated/reserved - * memory. - */ -void __init dma_contiguous_reserve(phys_addr_t limit) -{ - phys_addr_t selected_size = 0; - - pr_debug("%s(limit %08lx)\n", __func__, (unsigned long)limit); - - if (size_cmdline != -1) { - selected_size = size_cmdline; - } else { -#ifdef CONFIG_CMA_SIZE_SEL_MBYTES - selected_size = size_bytes; -#elif defined(CONFIG_CMA_SIZE_SEL_PERCENTAGE) - selected_size = cma_early_percent_memory(); -#elif defined(CONFIG_CMA_SIZE_SEL_MIN) - selected_size = min(size_bytes, cma_early_percent_memory()); -#elif defined(CONFIG_CMA_SIZE_SEL_MAX) - selected_size = max(size_bytes, cma_early_percent_memory()); -#endif - } - - if (selected_size) { - pr_debug("%s: reserving %ld MiB for global area\n", __func__, - (unsigned long)selected_size / SZ_1M); - - dma_declare_contiguous(NULL, selected_size, 0, limit); - } -}; - -static DEFINE_MUTEX(cma_mutex); - -static __init int cma_activate_area(unsigned long base_pfn, unsigned long count) -{ - unsigned long pfn = base_pfn; - unsigned i = count >> pageblock_order; - struct zone *zone; - - WARN_ON_ONCE(!pfn_valid(pfn)); - zone = page_zone(pfn_to_page(pfn)); - - do { - unsigned j; - base_pfn = pfn; - for (j = pageblock_nr_pages; j; --j, pfn++) { - WARN_ON_ONCE(!pfn_valid(pfn)); - if (page_zone(pfn_to_page(pfn)) != zone) - return -EINVAL; - } - init_cma_reserved_pageblock(pfn_to_page(base_pfn)); - } while (--i); - return 0; -} - -static __init struct cma *cma_create_area(unsigned long base_pfn, - unsigned long count) -{ - int bitmap_size = BITS_TO_LONGS(count) * sizeof(long); - struct cma *cma; - int ret = -ENOMEM; - - pr_debug("%s(base %08lx, count %lx)\n", __func__, base_pfn, count); - - cma = kmalloc(sizeof *cma, GFP_KERNEL); - if (!cma) - return ERR_PTR(-ENOMEM); - - cma->base_pfn = base_pfn; - cma->count = count; - cma->bitmap = kzalloc(bitmap_size, GFP_KERNEL); - - if (!cma->bitmap) - goto no_mem; - - ret = cma_activate_area(base_pfn, count); - if (ret) - goto error; - - pr_debug("%s: returned %p\n", __func__, (void *)cma); - return cma; - -error: - kfree(cma->bitmap); -no_mem: - kfree(cma); - return ERR_PTR(ret); -} - -static struct cma_reserved { - phys_addr_t start; - unsigned long size; - struct device *dev; -} cma_reserved[MAX_CMA_AREAS] __initdata; -static unsigned cma_reserved_count __initdata; - -static int __init cma_init_reserved_areas(void) -{ - struct cma_reserved *r = cma_reserved; - unsigned i = cma_reserved_count; - - pr_debug("%s()\n", __func__); - - for (; i; --i, ++r) { - struct cma *cma; - cma = cma_create_area(PFN_DOWN(r->start), - r->size >> PAGE_SHIFT); - if (!IS_ERR(cma)) - dev_set_cma_area(r->dev, cma); - } - return 0; -} -core_initcall(cma_init_reserved_areas); - -/** - * dma_declare_contiguous() - reserve area for contiguous memory handling - * for particular device - * @dev: Pointer to device structure. - * @size: Size of the reserved memory. - * @base: Start address of the reserved memory (optional, 0 for any). - * @limit: End address of the reserved memory (optional, 0 for any). - * - * This function reserves memory for specified device. It should be - * called by board specific code when early allocator (memblock or bootmem) - * is still activate. - */ -int __init dma_declare_contiguous(struct device *dev, phys_addr_t size, - phys_addr_t base, phys_addr_t limit) -{ - struct cma_reserved *r = &cma_reserved[cma_reserved_count]; - phys_addr_t alignment; - - pr_debug("%s(size %lx, base %08lx, limit %08lx)\n", __func__, - (unsigned long)size, (unsigned long)base, - (unsigned long)limit); - - /* Sanity checks */ - if (cma_reserved_count == ARRAY_SIZE(cma_reserved)) { - pr_err("Not enough slots for CMA reserved regions!\n"); - return -ENOSPC; - } - - if (!size) - return -EINVAL; - - /* Sanitise input arguments */ - alignment = PAGE_SIZE << max(MAX_ORDER - 1, pageblock_order); - base = ALIGN(base, alignment); - size = ALIGN(size, alignment); - limit &= ~(alignment - 1); - - /* Reserve memory */ - if (base) { - if (memblock_is_region_reserved(base, size) || - memblock_reserve(base, size) < 0) { - base = -EBUSY; - goto err; - } - } else { - /* - * Use __memblock_alloc_base() since - * memblock_alloc_base() panic()s. - */ - phys_addr_t addr = __memblock_alloc_base(size, alignment, limit); - if (!addr) { - base = -ENOMEM; - goto err; - } else { - base = addr; - } - } - - /* - * Each reserved area must be initialised later, when more kernel - * subsystems (like slab allocator) are available. - */ - r->start = base; - r->size = size; - r->dev = dev; - cma_reserved_count++; - pr_info("CMA: reserved %ld MiB at %08lx\n", (unsigned long)size / SZ_1M, - (unsigned long)base); - - /* Architecture specific contiguous memory fixup. */ - dma_contiguous_early_fixup(base, size); - return 0; -err: - pr_err("CMA: failed to reserve %ld MiB\n", (unsigned long)size / SZ_1M); - return base; -} - -/** - * dma_alloc_from_contiguous() - allocate pages from contiguous area - * @dev: Pointer to device for which the allocation is performed. - * @count: Requested number of pages. - * @align: Requested alignment of pages (in PAGE_SIZE order). - * - * This function allocates memory buffer for specified device. It uses - * device specific contiguous memory area if available or the default - * global one. Requires architecture specific get_dev_cma_area() helper - * function. - */ -struct page *dma_alloc_from_contiguous(struct device *dev, int count, - unsigned int align) -{ - unsigned long mask, pfn, pageno, start = 0; - struct cma *cma = dev_get_cma_area(dev); - struct page *page = NULL; - int ret; - - if (!cma || !cma->count) - return NULL; - - if (align > CONFIG_CMA_ALIGNMENT) - align = CONFIG_CMA_ALIGNMENT; - - pr_debug("%s(cma %p, count %d, align %d)\n", __func__, (void *)cma, - count, align); - - if (!count) - return NULL; - - mask = (1 << align) - 1; - - mutex_lock(&cma_mutex); - - for (;;) { - pageno = bitmap_find_next_zero_area(cma->bitmap, cma->count, - start, count, mask); - if (pageno >= cma->count) - break; - - pfn = cma->base_pfn + pageno; - ret = alloc_contig_range(pfn, pfn + count, MIGRATE_CMA); - if (ret == 0) { - bitmap_set(cma->bitmap, pageno, count); - page = pfn_to_page(pfn); - break; - } else if (ret != -EBUSY) { - break; - } - pr_debug("%s(): memory range at %p is busy, retrying\n", - __func__, pfn_to_page(pfn)); - /* try again with a bit different memory target */ - start = pageno + mask + 1; - } - - mutex_unlock(&cma_mutex); - pr_debug("%s(): returned %p\n", __func__, page); - return page; -} - -/** - * dma_release_from_contiguous() - release allocated pages - * @dev: Pointer to device for which the pages were allocated. - * @pages: Allocated pages. - * @count: Number of allocated pages. - * - * This function releases memory allocated by dma_alloc_from_contiguous(). - * It returns false when provided pages do not belong to contiguous area and - * true otherwise. - */ -bool dma_release_from_contiguous(struct device *dev, struct page *pages, - int count) -{ - struct cma *cma = dev_get_cma_area(dev); - unsigned long pfn; - - if (!cma || !pages) - return false; - - pr_debug("%s(page %p)\n", __func__, (void *)pages); - - pfn = page_to_pfn(pages); - - if (pfn < cma->base_pfn || pfn >= cma->base_pfn + cma->count) - return false; - - VM_BUG_ON(pfn + count > cma->base_pfn + cma->count); - - mutex_lock(&cma_mutex); - bitmap_clear(cma->bitmap, pfn - cma->base_pfn, count); - free_contig_range(pfn, count); - mutex_unlock(&cma_mutex); - - return true; -} diff --git a/drivers/base/dma-mapping.c b/drivers/base/dma-mapping.c deleted file mode 100644 index 0ce39a33b3c2..000000000000 --- a/drivers/base/dma-mapping.c +++ /dev/null @@ -1,269 +0,0 @@ -/* - * drivers/base/dma-mapping.c - arch-independent dma-mapping routines - * - * Copyright (c) 2006 SUSE Linux Products GmbH - * Copyright (c) 2006 Tejun Heo <teheo@suse.de> - * - * This file is released under the GPLv2. - */ - -#include <linux/dma-mapping.h> -#include <linux/export.h> -#include <linux/gfp.h> -#include <asm-generic/dma-coherent.h> - -/* - * Managed DMA API - */ -struct dma_devres { - size_t size; - void *vaddr; - dma_addr_t dma_handle; -}; - -static void dmam_coherent_release(struct device *dev, void *res) -{ - struct dma_devres *this = res; - - dma_free_coherent(dev, this->size, this->vaddr, this->dma_handle); -} - -static void dmam_noncoherent_release(struct device *dev, void *res) -{ - struct dma_devres *this = res; - - dma_free_noncoherent(dev, this->size, this->vaddr, this->dma_handle); -} - -static int dmam_match(struct device *dev, void *res, void *match_data) -{ - struct dma_devres *this = res, *match = match_data; - - if (this->vaddr == match->vaddr) { - WARN_ON(this->size != match->size || - this->dma_handle != match->dma_handle); - return 1; - } - return 0; -} - -/** - * dmam_alloc_coherent - Managed dma_alloc_coherent() - * @dev: Device to allocate coherent memory for - * @size: Size of allocation - * @dma_handle: Out argument for allocated DMA handle - * @gfp: Allocation flags - * - * Managed dma_alloc_coherent(). Memory allocated using this function - * will be automatically released on driver detach. - * - * RETURNS: - * Pointer to allocated memory on success, NULL on failure. - */ -void * dmam_alloc_coherent(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t gfp) -{ - struct dma_devres *dr; - void *vaddr; - - dr = devres_alloc(dmam_coherent_release, sizeof(*dr), gfp); - if (!dr) - return NULL; - - vaddr = dma_alloc_coherent(dev, size, dma_handle, gfp); - if (!vaddr) { - devres_free(dr); - return NULL; - } - - dr->vaddr = vaddr; - dr->dma_handle = *dma_handle; - dr->size = size; - - devres_add(dev, dr); - - return vaddr; -} -EXPORT_SYMBOL(dmam_alloc_coherent); - -/** - * dmam_free_coherent - Managed dma_free_coherent() - * @dev: Device to free coherent memory for - * @size: Size of allocation - * @vaddr: Virtual address of the memory to free - * @dma_handle: DMA handle of the memory to free - * - * Managed dma_free_coherent(). - */ -void dmam_free_coherent(struct device *dev, size_t size, void *vaddr, - dma_addr_t dma_handle) -{ - struct dma_devres match_data = { size, vaddr, dma_handle }; - - dma_free_coherent(dev, size, vaddr, dma_handle); - WARN_ON(devres_destroy(dev, dmam_coherent_release, dmam_match, - &match_data)); -} -EXPORT_SYMBOL(dmam_free_coherent); - -/** - * dmam_alloc_non_coherent - Managed dma_alloc_non_coherent() - * @dev: Device to allocate non_coherent memory for - * @size: Size of allocation - * @dma_handle: Out argument for allocated DMA handle - * @gfp: Allocation flags - * - * Managed dma_alloc_non_coherent(). Memory allocated using this - * function will be automatically released on driver detach. - * - * RETURNS: - * Pointer to allocated memory on success, NULL on failure. - */ -void *dmam_alloc_noncoherent(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t gfp) -{ - struct dma_devres *dr; - void *vaddr; - - dr = devres_alloc(dmam_noncoherent_release, sizeof(*dr), gfp); - if (!dr) - return NULL; - - vaddr = dma_alloc_noncoherent(dev, size, dma_handle, gfp); - if (!vaddr) { - devres_free(dr); - return NULL; - } - - dr->vaddr = vaddr; - dr->dma_handle = *dma_handle; - dr->size = size; - - devres_add(dev, dr); - - return vaddr; -} -EXPORT_SYMBOL(dmam_alloc_noncoherent); - -/** - * dmam_free_coherent - Managed dma_free_noncoherent() - * @dev: Device to free noncoherent memory for - * @size: Size of allocation - * @vaddr: Virtual address of the memory to free - * @dma_handle: DMA handle of the memory to free - * - * Managed dma_free_noncoherent(). - */ -void dmam_free_noncoherent(struct device *dev, size_t size, void *vaddr, - dma_addr_t dma_handle) -{ - struct dma_devres match_data = { size, vaddr, dma_handle }; - - dma_free_noncoherent(dev, size, vaddr, dma_handle); - WARN_ON(!devres_destroy(dev, dmam_noncoherent_release, dmam_match, - &match_data)); -} -EXPORT_SYMBOL(dmam_free_noncoherent); - -#ifdef ARCH_HAS_DMA_DECLARE_COHERENT_MEMORY - -static void dmam_coherent_decl_release(struct device *dev, void *res) -{ - dma_release_declared_memory(dev); -} - -/** - * dmam_declare_coherent_memory - Managed dma_declare_coherent_memory() - * @dev: Device to declare coherent memory for - * @bus_addr: Bus address of coherent memory to be declared - * @device_addr: Device address of coherent memory to be declared - * @size: Size of coherent memory to be declared - * @flags: Flags - * - * Managed dma_declare_coherent_memory(). - * - * RETURNS: - * 0 on success, -errno on failure. - */ -int dmam_declare_coherent_memory(struct device *dev, dma_addr_t bus_addr, - dma_addr_t device_addr, size_t size, int flags) -{ - void *res; - int rc; - - res = devres_alloc(dmam_coherent_decl_release, 0, GFP_KERNEL); - if (!res) - return -ENOMEM; - - rc = dma_declare_coherent_memory(dev, bus_addr, device_addr, size, - flags); - if (rc == 0) - devres_add(dev, res); - else - devres_free(res); - - return rc; -} -EXPORT_SYMBOL(dmam_declare_coherent_memory); - -/** - * dmam_release_declared_memory - Managed dma_release_declared_memory(). - * @dev: Device to release declared coherent memory for - * - * Managed dmam_release_declared_memory(). - */ -void dmam_release_declared_memory(struct device *dev) -{ - WARN_ON(devres_destroy(dev, dmam_coherent_decl_release, NULL, NULL)); -} -EXPORT_SYMBOL(dmam_release_declared_memory); - -#endif - -/* - * Create scatter-list for the already allocated DMA buffer. - */ -int dma_common_get_sgtable(struct device *dev, struct sg_table *sgt, - void *cpu_addr, dma_addr_t handle, size_t size) -{ - struct page *page = virt_to_page(cpu_addr); - int ret; - - ret = sg_alloc_table(sgt, 1, GFP_KERNEL); - if (unlikely(ret)) - return ret; - - sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0); - return 0; -} -EXPORT_SYMBOL(dma_common_get_sgtable); - -/* - * Create userspace mapping for the DMA-coherent memory. - */ -int dma_common_mmap(struct device *dev, struct vm_area_struct *vma, - void *cpu_addr, dma_addr_t dma_addr, size_t size) -{ - int ret = -ENXIO; -#ifdef CONFIG_MMU - unsigned long user_count = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; - unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT; - unsigned long pfn = page_to_pfn(virt_to_page(cpu_addr)); - unsigned long off = vma->vm_pgoff; - - vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); - - if (dma_mmap_from_coherent(dev, vma, cpu_addr, size, &ret)) - return ret; - - if (off < count && user_count <= (count - off)) { - ret = remap_pfn_range(vma, vma->vm_start, - pfn + off, - user_count << PAGE_SHIFT, - vma->vm_page_prot); - } -#endif /* CONFIG_MMU */ - - return ret; -} -EXPORT_SYMBOL(dma_common_mmap); diff --git a/drivers/base/driver.c b/drivers/base/driver.c index 974e301a1ef0..8ab010ddf709 100644 --- a/drivers/base/driver.c +++ b/drivers/base/driver.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * driver.c - centralized device driver management * @@ -5,16 +6,15 @@ * Copyright (c) 2002-3 Open Source Development Labs * Copyright (c) 2007 Greg Kroah-Hartman <gregkh@suse.de> * Copyright (c) 2007 Novell Inc. - * - * This file is released under the GPLv2 - * */ +#include <linux/device/driver.h> #include <linux/device.h> #include <linux/module.h> #include <linux/errno.h> #include <linux/slab.h> #include <linux/string.h> +#include <linux/sysfs.h> #include "base.h" static struct device *next_device(struct klist_iter *i) @@ -31,6 +31,81 @@ static struct device *next_device(struct klist_iter *i) } /** + * driver_set_override() - Helper to set or clear driver override. + * @dev: Device to change + * @override: Address of string to change (e.g. &device->driver_override); + * The contents will be freed and hold newly allocated override. + * @s: NUL-terminated string, new driver name to force a match, pass empty + * string to clear it ("" or "\n", where the latter is only for sysfs + * interface). + * @len: length of @s + * + * Helper to set or clear driver override in a device, intended for the cases + * when the driver_override field is allocated by driver/bus code. + * + * Returns: 0 on success or a negative error code on failure. + */ +int driver_set_override(struct device *dev, const char **override, + const char *s, size_t len) +{ + const char *new, *old; + char *cp; + + if (!override || !s) + return -EINVAL; + + /* + * The stored value will be used in sysfs show callback (sysfs_emit()), + * which has a length limit of PAGE_SIZE and adds a trailing newline. + * Thus we can store one character less to avoid truncation during sysfs + * show. + */ + if (len >= (PAGE_SIZE - 1)) + return -EINVAL; + + /* + * Compute the real length of the string in case userspace sends us a + * bunch of \0 characters like python likes to do. + */ + len = strlen(s); + + if (!len) { + /* Empty string passed - clear override */ + device_lock(dev); + old = *override; + *override = NULL; + device_unlock(dev); + kfree(old); + + return 0; + } + + cp = strnchr(s, len, '\n'); + if (cp) + len = cp - s; + + new = kstrndup(s, len, GFP_KERNEL); + if (!new) + return -ENOMEM; + + device_lock(dev); + old = *override; + if (cp != s) { + *override = new; + } else { + /* "\n" passed - clear override */ + kfree(new); + *override = NULL; + } + device_unlock(dev); + + kfree(old); + + return 0; +} +EXPORT_SYMBOL_GPL(driver_set_override); + +/** * driver_for_each_device - Iterator for devices bound to a driver. * @drv: Driver we're iterating. * @start: Device to begin with @@ -40,7 +115,7 @@ static struct device *next_device(struct klist_iter *i) * Iterate over the @drv's list of devices calling @fn for each one. */ int driver_for_each_device(struct device_driver *drv, struct device *start, - void *data, int (*fn)(struct device *, void *)) + void *data, device_iter_t fn) { struct klist_iter i; struct device *dev; @@ -51,7 +126,7 @@ int driver_for_each_device(struct device_driver *drv, struct device *start, klist_iter_init_node(&drv->p->klist_devices, &i, start ? &start->p->knode_driver : NULL); - while ((dev = next_device(&i)) && !error) + while (!error && (dev = next_device(&i))) error = fn(dev, data); klist_iter_exit(&i); return error; @@ -73,9 +148,9 @@ EXPORT_SYMBOL_GPL(driver_for_each_device); * if it does. If the callback returns non-zero, this function will * return to the caller and not iterate over any more devices. */ -struct device *driver_find_device(struct device_driver *drv, - struct device *start, void *data, - int (*match)(struct device *dev, void *data)) +struct device *driver_find_device(const struct device_driver *drv, + struct device *start, const void *data, + device_match_t match) { struct klist_iter i; struct device *dev; @@ -85,9 +160,12 @@ struct device *driver_find_device(struct device_driver *drv, klist_iter_init_node(&drv->p->klist_devices, &i, (start ? &start->p->knode_driver : NULL)); - while ((dev = next_device(&i))) - if (match(dev, data) && get_device(dev)) + while ((dev = next_device(&i))) { + if (match(dev, data)) { + get_device(dev); break; + } + } klist_iter_exit(&i); return dev; } @@ -98,10 +176,11 @@ EXPORT_SYMBOL_GPL(driver_find_device); * @drv: driver. * @attr: driver attribute descriptor. */ -int driver_create_file(struct device_driver *drv, +int driver_create_file(const struct device_driver *drv, const struct driver_attribute *attr) { int error; + if (drv) error = sysfs_create_file(&drv->p->kobj, &attr->attr); else @@ -115,7 +194,7 @@ EXPORT_SYMBOL_GPL(driver_create_file); * @drv: driver. * @attr: driver attribute descriptor. */ -void driver_remove_file(struct device_driver *drv, +void driver_remove_file(const struct device_driver *drv, const struct driver_attribute *attr) { if (drv) @@ -123,34 +202,16 @@ void driver_remove_file(struct device_driver *drv, } EXPORT_SYMBOL_GPL(driver_remove_file); -static int driver_add_groups(struct device_driver *drv, - const struct attribute_group **groups) +int driver_add_groups(const struct device_driver *drv, + const struct attribute_group **groups) { - int error = 0; - int i; - - if (groups) { - for (i = 0; groups[i]; i++) { - error = sysfs_create_group(&drv->p->kobj, groups[i]); - if (error) { - while (--i >= 0) - sysfs_remove_group(&drv->p->kobj, - groups[i]); - break; - } - } - } - return error; + return sysfs_create_groups(&drv->p->kobj, groups); } -static void driver_remove_groups(struct device_driver *drv, - const struct attribute_group **groups) +void driver_remove_groups(const struct device_driver *drv, + const struct attribute_group **groups) { - int i; - - if (groups) - for (i = 0; groups[i]; i++) - sysfs_remove_group(&drv->p->kobj, groups[i]); + sysfs_remove_groups(&drv->p->kobj, groups); } /** @@ -166,17 +227,21 @@ int driver_register(struct device_driver *drv) int ret; struct device_driver *other; - BUG_ON(!drv->bus->p); + if (!bus_is_registered(drv->bus)) { + pr_err("Driver '%s' was unable to register with bus_type '%s' because the bus was not initialized.\n", + drv->name, drv->bus->name); + return -EINVAL; + } if ((drv->bus->probe && drv->probe) || (drv->bus->remove && drv->remove) || (drv->bus->shutdown && drv->shutdown)) - printk(KERN_WARNING "Driver '%s' needs updating - please use " + pr_warn("Driver '%s' needs updating - please use " "bus_type methods\n", drv->name); other = driver_find(drv->name, drv->bus); if (other) { - printk(KERN_ERR "Error: Driver '%s' is already registered, " + pr_err("Error: Driver '%s' is already registered, " "aborting...\n", drv->name); return -EBUSY; } @@ -190,6 +255,7 @@ int driver_register(struct device_driver *drv) return ret; } kobject_uevent(&drv->p->kobj, KOBJ_ADD); + deferred_probe_extend_timeout(); return ret; } @@ -211,30 +277,3 @@ void driver_unregister(struct device_driver *drv) bus_remove_driver(drv); } EXPORT_SYMBOL_GPL(driver_unregister); - -/** - * driver_find - locate driver on a bus by its name. - * @name: name of the driver. - * @bus: bus to scan for the driver. - * - * Call kset_find_obj() to iterate over list of drivers on - * a bus to find driver by name. Return driver if found. - * - * This routine provides no locking to prevent the driver it returns - * from being unregistered or unloaded while the caller is using it. - * The caller is responsible for preventing this. - */ -struct device_driver *driver_find(const char *name, struct bus_type *bus) -{ - struct kobject *k = kset_find_obj(bus->p->drivers_kset, name); - struct driver_private *priv; - - if (k) { - /* Drop reference added by kset_find_obj() */ - kobject_put(k); - priv = to_driver(k); - return priv->driver; - } - return NULL; -} -EXPORT_SYMBOL_GPL(driver_find); diff --git a/drivers/base/faux.c b/drivers/base/faux.c new file mode 100644 index 000000000000..21dd02124231 --- /dev/null +++ b/drivers/base/faux.c @@ -0,0 +1,261 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2025 Greg Kroah-Hartman <gregkh@linuxfoundation.org> + * Copyright (c) 2025 The Linux Foundation + * + * A "simple" faux bus that allows devices to be created and added + * automatically to it. This is to be used whenever you need to create a + * device that is not associated with any "real" system resources, and do + * not want to have to deal with a bus/driver binding logic. It is + * intended to be very simple, with only a create and a destroy function + * available. + */ +#include <linux/err.h> +#include <linux/init.h> +#include <linux/slab.h> +#include <linux/string.h> +#include <linux/container_of.h> +#include <linux/device/faux.h> +#include "base.h" + +/* + * Internal wrapper structure so we can hold a pointer to the + * faux_device_ops for this device. + */ +struct faux_object { + struct faux_device faux_dev; + const struct faux_device_ops *faux_ops; + const struct attribute_group **groups; +}; +#define to_faux_object(dev) container_of_const(dev, struct faux_object, faux_dev.dev) + +static struct device faux_bus_root = { + .init_name = "faux", +}; + +static int faux_match(struct device *dev, const struct device_driver *drv) +{ + /* Match always succeeds, we only have one driver */ + return 1; +} + +static int faux_probe(struct device *dev) +{ + struct faux_object *faux_obj = to_faux_object(dev); + struct faux_device *faux_dev = &faux_obj->faux_dev; + const struct faux_device_ops *faux_ops = faux_obj->faux_ops; + int ret; + + if (faux_ops && faux_ops->probe) { + ret = faux_ops->probe(faux_dev); + if (ret) + return ret; + } + + /* + * Add groups after the probe succeeds to ensure resources are + * initialized correctly + */ + ret = device_add_groups(dev, faux_obj->groups); + if (ret && faux_ops && faux_ops->remove) + faux_ops->remove(faux_dev); + + return ret; +} + +static void faux_remove(struct device *dev) +{ + struct faux_object *faux_obj = to_faux_object(dev); + struct faux_device *faux_dev = &faux_obj->faux_dev; + const struct faux_device_ops *faux_ops = faux_obj->faux_ops; + + device_remove_groups(dev, faux_obj->groups); + + if (faux_ops && faux_ops->remove) + faux_ops->remove(faux_dev); +} + +static const struct bus_type faux_bus_type = { + .name = "faux", + .match = faux_match, + .probe = faux_probe, + .remove = faux_remove, +}; + +static struct device_driver faux_driver = { + .name = "faux_driver", + .bus = &faux_bus_type, + .probe_type = PROBE_FORCE_SYNCHRONOUS, + .suppress_bind_attrs = true, +}; + +static void faux_device_release(struct device *dev) +{ + struct faux_object *faux_obj = to_faux_object(dev); + + kfree(faux_obj); +} + +/** + * faux_device_create_with_groups - Create and register with the driver + * core a faux device and populate the device with an initial + * set of sysfs attributes. + * @name: The name of the device we are adding, must be unique for + * all faux devices. + * @parent: Pointer to a potential parent struct device. If set to + * NULL, the device will be created in the "root" of the faux + * device tree in sysfs. + * @faux_ops: struct faux_device_ops that the new device will call back + * into, can be NULL. + * @groups: The set of sysfs attributes that will be created for this + * device when it is registered with the driver core. + * + * Create a new faux device and register it in the driver core properly. + * If present, callbacks in @faux_ops will be called with the device that + * for the caller to do something with at the proper time given the + * device's lifecycle. + * + * Note, when this function is called, the functions specified in struct + * faux_ops can be called before the function returns, so be prepared for + * everything to be properly initialized before that point in time. If the + * probe callback (if one is present) does NOT succeed, the creation of the + * device will fail and NULL will be returned. + * + * Return: + * * NULL if an error happened with creating the device + * * pointer to a valid struct faux_device that is registered with sysfs + */ +struct faux_device *faux_device_create_with_groups(const char *name, + struct device *parent, + const struct faux_device_ops *faux_ops, + const struct attribute_group **groups) +{ + struct faux_object *faux_obj; + struct faux_device *faux_dev; + struct device *dev; + int ret; + + faux_obj = kzalloc(sizeof(*faux_obj), GFP_KERNEL); + if (!faux_obj) + return NULL; + + /* Save off the callbacks and groups so we can use them in the future */ + faux_obj->faux_ops = faux_ops; + faux_obj->groups = groups; + + /* Initialize the device portion and register it with the driver core */ + faux_dev = &faux_obj->faux_dev; + dev = &faux_dev->dev; + + device_initialize(dev); + dev->release = faux_device_release; + if (parent) + dev->parent = parent; + else + dev->parent = &faux_bus_root; + dev->bus = &faux_bus_type; + dev_set_name(dev, "%s", name); + device_set_pm_not_required(dev); + + ret = device_add(dev); + if (ret) { + pr_err("%s: device_add for faux device '%s' failed with %d\n", + __func__, name, ret); + put_device(dev); + return NULL; + } + + /* + * Verify that we did bind the driver to the device (i.e. probe worked), + * if not, let's fail the creation as trying to guess if probe was + * successful is almost impossible to determine by the caller. + */ + if (!dev->driver) { + dev_dbg(dev, "probe did not succeed, tearing down the device\n"); + faux_device_destroy(faux_dev); + faux_dev = NULL; + } + + return faux_dev; +} +EXPORT_SYMBOL_GPL(faux_device_create_with_groups); + +/** + * faux_device_create - create and register with the driver core a faux device + * @name: The name of the device we are adding, must be unique for all + * faux devices. + * @parent: Pointer to a potential parent struct device. If set to + * NULL, the device will be created in the "root" of the faux + * device tree in sysfs. + * @faux_ops: struct faux_device_ops that the new device will call back + * into, can be NULL. + * + * Create a new faux device and register it in the driver core properly. + * If present, callbacks in @faux_ops will be called with the device that + * for the caller to do something with at the proper time given the + * device's lifecycle. + * + * Note, when this function is called, the functions specified in struct + * faux_ops can be called before the function returns, so be prepared for + * everything to be properly initialized before that point in time. + * + * Return: + * * NULL if an error happened with creating the device + * * pointer to a valid struct faux_device that is registered with sysfs + */ +struct faux_device *faux_device_create(const char *name, + struct device *parent, + const struct faux_device_ops *faux_ops) +{ + return faux_device_create_with_groups(name, parent, faux_ops, NULL); +} +EXPORT_SYMBOL_GPL(faux_device_create); + +/** + * faux_device_destroy - destroy a faux device + * @faux_dev: faux device to destroy + * + * Unregisters and cleans up a device that was created with a call to + * faux_device_create() + */ +void faux_device_destroy(struct faux_device *faux_dev) +{ + struct device *dev = &faux_dev->dev; + + if (!faux_dev) + return; + + device_del(dev); + + /* The final put_device() will clean up the memory we allocated for this device. */ + put_device(dev); +} +EXPORT_SYMBOL_GPL(faux_device_destroy); + +int __init faux_bus_init(void) +{ + int ret; + + ret = device_register(&faux_bus_root); + if (ret) { + put_device(&faux_bus_root); + return ret; + } + + ret = bus_register(&faux_bus_type); + if (ret) + goto error_bus; + + ret = driver_register(&faux_driver); + if (ret) + goto error_driver; + + return ret; + +error_driver: + bus_unregister(&faux_bus_type); + +error_bus: + device_unregister(&faux_bus_root); + return ret; +} diff --git a/drivers/base/firmware.c b/drivers/base/firmware.c index 113815556809..8dff940e0db9 100644 --- a/drivers/base/firmware.c +++ b/drivers/base/firmware.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * firmware.c - firmware subsystem hoohaw. * @@ -5,8 +6,6 @@ * Copyright (c) 2002-3 Open Source Development Labs * Copyright (c) 2007 Greg Kroah-Hartman <gregkh@suse.de> * Copyright (c) 2007 Novell Inc. - * - * This file is released under the GPLv2 */ #include <linux/kobject.h> #include <linux/module.h> diff --git a/drivers/base/firmware_class.c b/drivers/base/firmware_class.c deleted file mode 100644 index a439602ea919..000000000000 --- a/drivers/base/firmware_class.c +++ /dev/null @@ -1,1615 +0,0 @@ -/* - * firmware_class.c - Multi purpose firmware loading support - * - * Copyright (c) 2003 Manuel Estrada Sainz - * - * Please see Documentation/firmware_class/ for more information. - * - */ - -#include <linux/capability.h> -#include <linux/device.h> -#include <linux/module.h> -#include <linux/init.h> -#include <linux/timer.h> -#include <linux/vmalloc.h> -#include <linux/interrupt.h> -#include <linux/bitops.h> -#include <linux/mutex.h> -#include <linux/workqueue.h> -#include <linux/highmem.h> -#include <linux/firmware.h> -#include <linux/slab.h> -#include <linux/sched.h> -#include <linux/file.h> -#include <linux/list.h> -#include <linux/async.h> -#include <linux/pm.h> -#include <linux/suspend.h> -#include <linux/syscore_ops.h> -#include <linux/reboot.h> - -#include <generated/utsrelease.h> - -#include "base.h" - -MODULE_AUTHOR("Manuel Estrada Sainz"); -MODULE_DESCRIPTION("Multi purpose firmware loading support"); -MODULE_LICENSE("GPL"); - -/* Builtin firmware support */ - -#ifdef CONFIG_FW_LOADER - -extern struct builtin_fw __start_builtin_fw[]; -extern struct builtin_fw __end_builtin_fw[]; - -static bool fw_get_builtin_firmware(struct firmware *fw, const char *name) -{ - struct builtin_fw *b_fw; - - for (b_fw = __start_builtin_fw; b_fw != __end_builtin_fw; b_fw++) { - if (strcmp(name, b_fw->name) == 0) { - fw->size = b_fw->size; - fw->data = b_fw->data; - return true; - } - } - - return false; -} - -static bool fw_is_builtin_firmware(const struct firmware *fw) -{ - struct builtin_fw *b_fw; - - for (b_fw = __start_builtin_fw; b_fw != __end_builtin_fw; b_fw++) - if (fw->data == b_fw->data) - return true; - - return false; -} - -#else /* Module case - no builtin firmware support */ - -static inline bool fw_get_builtin_firmware(struct firmware *fw, const char *name) -{ - return false; -} - -static inline bool fw_is_builtin_firmware(const struct firmware *fw) -{ - return false; -} -#endif - -enum { - FW_STATUS_LOADING, - FW_STATUS_DONE, - FW_STATUS_ABORT, -}; - -static int loading_timeout = 60; /* In seconds */ - -static inline long firmware_loading_timeout(void) -{ - return loading_timeout > 0 ? loading_timeout * HZ : MAX_SCHEDULE_TIMEOUT; -} - -struct firmware_cache { - /* firmware_buf instance will be added into the below list */ - spinlock_t lock; - struct list_head head; - int state; - -#ifdef CONFIG_PM_SLEEP - /* - * Names of firmware images which have been cached successfully - * will be added into the below list so that device uncache - * helper can trace which firmware images have been cached - * before. - */ - spinlock_t name_lock; - struct list_head fw_names; - - struct delayed_work work; - - struct notifier_block pm_notify; -#endif -}; - -struct firmware_buf { - struct kref ref; - struct list_head list; - struct completion completion; - struct firmware_cache *fwc; - unsigned long status; - void *data; - size_t size; -#ifdef CONFIG_FW_LOADER_USER_HELPER - bool is_paged_buf; - bool need_uevent; - struct page **pages; - int nr_pages; - int page_array_size; - struct list_head pending_list; -#endif - char fw_id[]; -}; - -struct fw_cache_entry { - struct list_head list; - char name[]; -}; - -struct fw_name_devm { - unsigned long magic; - char name[]; -}; - -#define to_fwbuf(d) container_of(d, struct firmware_buf, ref) - -#define FW_LOADER_NO_CACHE 0 -#define FW_LOADER_START_CACHE 1 - -static int fw_cache_piggyback_on_request(const char *name); - -/* fw_lock could be moved to 'struct firmware_priv' but since it is just - * guarding for corner cases a global lock should be OK */ -static DEFINE_MUTEX(fw_lock); - -static struct firmware_cache fw_cache; - -static struct firmware_buf *__allocate_fw_buf(const char *fw_name, - struct firmware_cache *fwc) -{ - struct firmware_buf *buf; - - buf = kzalloc(sizeof(*buf) + strlen(fw_name) + 1 , GFP_ATOMIC); - - if (!buf) - return buf; - - kref_init(&buf->ref); - strcpy(buf->fw_id, fw_name); - buf->fwc = fwc; - init_completion(&buf->completion); -#ifdef CONFIG_FW_LOADER_USER_HELPER - INIT_LIST_HEAD(&buf->pending_list); -#endif - - pr_debug("%s: fw-%s buf=%p\n", __func__, fw_name, buf); - - return buf; -} - -static struct firmware_buf *__fw_lookup_buf(const char *fw_name) -{ - struct firmware_buf *tmp; - struct firmware_cache *fwc = &fw_cache; - - list_for_each_entry(tmp, &fwc->head, list) - if (!strcmp(tmp->fw_id, fw_name)) - return tmp; - return NULL; -} - -static int fw_lookup_and_allocate_buf(const char *fw_name, - struct firmware_cache *fwc, - struct firmware_buf **buf) -{ - struct firmware_buf *tmp; - - spin_lock(&fwc->lock); - tmp = __fw_lookup_buf(fw_name); - if (tmp) { - kref_get(&tmp->ref); - spin_unlock(&fwc->lock); - *buf = tmp; - return 1; - } - tmp = __allocate_fw_buf(fw_name, fwc); - if (tmp) - list_add(&tmp->list, &fwc->head); - spin_unlock(&fwc->lock); - - *buf = tmp; - - return tmp ? 0 : -ENOMEM; -} - -static void __fw_free_buf(struct kref *ref) -{ - struct firmware_buf *buf = to_fwbuf(ref); - struct firmware_cache *fwc = buf->fwc; - - pr_debug("%s: fw-%s buf=%p data=%p size=%u\n", - __func__, buf->fw_id, buf, buf->data, - (unsigned int)buf->size); - - list_del(&buf->list); - spin_unlock(&fwc->lock); - -#ifdef CONFIG_FW_LOADER_USER_HELPER - if (buf->is_paged_buf) { - int i; - vunmap(buf->data); - for (i = 0; i < buf->nr_pages; i++) - __free_page(buf->pages[i]); - kfree(buf->pages); - } else -#endif - vfree(buf->data); - kfree(buf); -} - -static void fw_free_buf(struct firmware_buf *buf) -{ - struct firmware_cache *fwc = buf->fwc; - spin_lock(&fwc->lock); - if (!kref_put(&buf->ref, __fw_free_buf)) - spin_unlock(&fwc->lock); -} - -/* direct firmware loading support */ -static char fw_path_para[256]; -static const char * const fw_path[] = { - fw_path_para, - "/lib/firmware/updates/" UTS_RELEASE, - "/lib/firmware/updates", - "/lib/firmware/" UTS_RELEASE, - "/lib/firmware" -}; - -/* - * Typical usage is that passing 'firmware_class.path=$CUSTOMIZED_PATH' - * from kernel command line because firmware_class is generally built in - * kernel instead of module. - */ -module_param_string(path, fw_path_para, sizeof(fw_path_para), 0644); -MODULE_PARM_DESC(path, "customized firmware image search path with a higher priority than default path"); - -/* Don't inline this: 'struct kstat' is biggish */ -static noinline_for_stack long fw_file_size(struct file *file) -{ - struct kstat st; - if (vfs_getattr(&file->f_path, &st)) - return -1; - if (!S_ISREG(st.mode)) - return -1; - if (st.size != (long)st.size) - return -1; - return st.size; -} - -static bool fw_read_file_contents(struct file *file, struct firmware_buf *fw_buf) -{ - long size; - char *buf; - - size = fw_file_size(file); - if (size <= 0) - return false; - buf = vmalloc(size); - if (!buf) - return false; - if (kernel_read(file, 0, buf, size) != size) { - vfree(buf); - return false; - } - fw_buf->data = buf; - fw_buf->size = size; - return true; -} - -static bool fw_get_filesystem_firmware(struct device *device, - struct firmware_buf *buf) -{ - int i; - bool success = false; - char *path = __getname(); - - for (i = 0; i < ARRAY_SIZE(fw_path); i++) { - struct file *file; - - /* skip the unset customized path */ - if (!fw_path[i][0]) - continue; - - snprintf(path, PATH_MAX, "%s/%s", fw_path[i], buf->fw_id); - - file = filp_open(path, O_RDONLY, 0); - if (IS_ERR(file)) - continue; - success = fw_read_file_contents(file, buf); - fput(file); - if (success) - break; - } - __putname(path); - - if (success) { - dev_dbg(device, "firmware: direct-loading firmware %s\n", - buf->fw_id); - mutex_lock(&fw_lock); - set_bit(FW_STATUS_DONE, &buf->status); - complete_all(&buf->completion); - mutex_unlock(&fw_lock); - } - - return success; -} - -/* firmware holds the ownership of pages */ -static void firmware_free_data(const struct firmware *fw) -{ - /* Loaded directly? */ - if (!fw->priv) { - vfree(fw->data); - return; - } - fw_free_buf(fw->priv); -} - -/* store the pages buffer info firmware from buf */ -static void fw_set_page_data(struct firmware_buf *buf, struct firmware *fw) -{ - fw->priv = buf; -#ifdef CONFIG_FW_LOADER_USER_HELPER - fw->pages = buf->pages; -#endif - fw->size = buf->size; - fw->data = buf->data; - - pr_debug("%s: fw-%s buf=%p data=%p size=%u\n", - __func__, buf->fw_id, buf, buf->data, - (unsigned int)buf->size); -} - -#ifdef CONFIG_PM_SLEEP -static void fw_name_devm_release(struct device *dev, void *res) -{ - struct fw_name_devm *fwn = res; - - if (fwn->magic == (unsigned long)&fw_cache) - pr_debug("%s: fw_name-%s devm-%p released\n", - __func__, fwn->name, res); -} - -static int fw_devm_match(struct device *dev, void *res, - void *match_data) -{ - struct fw_name_devm *fwn = res; - - return (fwn->magic == (unsigned long)&fw_cache) && - !strcmp(fwn->name, match_data); -} - -static struct fw_name_devm *fw_find_devm_name(struct device *dev, - const char *name) -{ - struct fw_name_devm *fwn; - - fwn = devres_find(dev, fw_name_devm_release, - fw_devm_match, (void *)name); - return fwn; -} - -/* add firmware name into devres list */ -static int fw_add_devm_name(struct device *dev, const char *name) -{ - struct fw_name_devm *fwn; - - fwn = fw_find_devm_name(dev, name); - if (fwn) - return 1; - - fwn = devres_alloc(fw_name_devm_release, sizeof(struct fw_name_devm) + - strlen(name) + 1, GFP_KERNEL); - if (!fwn) - return -ENOMEM; - - fwn->magic = (unsigned long)&fw_cache; - strcpy(fwn->name, name); - devres_add(dev, fwn); - - return 0; -} -#else -static int fw_add_devm_name(struct device *dev, const char *name) -{ - return 0; -} -#endif - - -/* - * user-mode helper code - */ -#ifdef CONFIG_FW_LOADER_USER_HELPER -struct firmware_priv { - struct delayed_work timeout_work; - bool nowait; - struct device dev; - struct firmware_buf *buf; - struct firmware *fw; -}; - -static struct firmware_priv *to_firmware_priv(struct device *dev) -{ - return container_of(dev, struct firmware_priv, dev); -} - -static void __fw_load_abort(struct firmware_buf *buf) -{ - /* - * There is a small window in which user can write to 'loading' - * between loading done and disappearance of 'loading' - */ - if (test_bit(FW_STATUS_DONE, &buf->status)) - return; - - list_del_init(&buf->pending_list); - set_bit(FW_STATUS_ABORT, &buf->status); - complete_all(&buf->completion); -} - -static void fw_load_abort(struct firmware_priv *fw_priv) -{ - struct firmware_buf *buf = fw_priv->buf; - - __fw_load_abort(buf); - - /* avoid user action after loading abort */ - fw_priv->buf = NULL; -} - -#define is_fw_load_aborted(buf) \ - test_bit(FW_STATUS_ABORT, &(buf)->status) - -static LIST_HEAD(pending_fw_head); - -/* reboot notifier for avoid deadlock with usermode_lock */ -static int fw_shutdown_notify(struct notifier_block *unused1, - unsigned long unused2, void *unused3) -{ - mutex_lock(&fw_lock); - while (!list_empty(&pending_fw_head)) - __fw_load_abort(list_first_entry(&pending_fw_head, - struct firmware_buf, - pending_list)); - mutex_unlock(&fw_lock); - return NOTIFY_DONE; -} - -static struct notifier_block fw_shutdown_nb = { - .notifier_call = fw_shutdown_notify, -}; - -static ssize_t firmware_timeout_show(struct class *class, - struct class_attribute *attr, - char *buf) -{ - return sprintf(buf, "%d\n", loading_timeout); -} - -/** - * firmware_timeout_store - set number of seconds to wait for firmware - * @class: device class pointer - * @attr: device attribute pointer - * @buf: buffer to scan for timeout value - * @count: number of bytes in @buf - * - * Sets the number of seconds to wait for the firmware. Once - * this expires an error will be returned to the driver and no - * firmware will be provided. - * - * Note: zero means 'wait forever'. - **/ -static ssize_t firmware_timeout_store(struct class *class, - struct class_attribute *attr, - const char *buf, size_t count) -{ - loading_timeout = simple_strtol(buf, NULL, 10); - if (loading_timeout < 0) - loading_timeout = 0; - - return count; -} - -static struct class_attribute firmware_class_attrs[] = { - __ATTR(timeout, S_IWUSR | S_IRUGO, - firmware_timeout_show, firmware_timeout_store), - __ATTR_NULL -}; - -static void fw_dev_release(struct device *dev) -{ - struct firmware_priv *fw_priv = to_firmware_priv(dev); - - kfree(fw_priv); -} - -static int firmware_uevent(struct device *dev, struct kobj_uevent_env *env) -{ - struct firmware_priv *fw_priv = to_firmware_priv(dev); - - if (add_uevent_var(env, "FIRMWARE=%s", fw_priv->buf->fw_id)) - return -ENOMEM; - if (add_uevent_var(env, "TIMEOUT=%i", loading_timeout)) - return -ENOMEM; - if (add_uevent_var(env, "ASYNC=%d", fw_priv->nowait)) - return -ENOMEM; - - return 0; -} - -static struct class firmware_class = { - .name = "firmware", - .class_attrs = firmware_class_attrs, - .dev_uevent = firmware_uevent, - .dev_release = fw_dev_release, -}; - -static ssize_t firmware_loading_show(struct device *dev, - struct device_attribute *attr, char *buf) -{ - struct firmware_priv *fw_priv = to_firmware_priv(dev); - int loading = 0; - - mutex_lock(&fw_lock); - if (fw_priv->buf) - loading = test_bit(FW_STATUS_LOADING, &fw_priv->buf->status); - mutex_unlock(&fw_lock); - - return sprintf(buf, "%d\n", loading); -} - -/* Some architectures don't have PAGE_KERNEL_RO */ -#ifndef PAGE_KERNEL_RO -#define PAGE_KERNEL_RO PAGE_KERNEL -#endif - -/* one pages buffer should be mapped/unmapped only once */ -static int fw_map_pages_buf(struct firmware_buf *buf) -{ - if (!buf->is_paged_buf) - return 0; - - if (buf->data) - vunmap(buf->data); - buf->data = vmap(buf->pages, buf->nr_pages, 0, PAGE_KERNEL_RO); - if (!buf->data) - return -ENOMEM; - return 0; -} - -/** - * firmware_loading_store - set value in the 'loading' control file - * @dev: device pointer - * @attr: device attribute pointer - * @buf: buffer to scan for loading control value - * @count: number of bytes in @buf - * - * The relevant values are: - * - * 1: Start a load, discarding any previous partial load. - * 0: Conclude the load and hand the data to the driver code. - * -1: Conclude the load with an error and discard any written data. - **/ -static ssize_t firmware_loading_store(struct device *dev, - struct device_attribute *attr, - const char *buf, size_t count) -{ - struct firmware_priv *fw_priv = to_firmware_priv(dev); - struct firmware_buf *fw_buf; - int loading = simple_strtol(buf, NULL, 10); - int i; - - mutex_lock(&fw_lock); - fw_buf = fw_priv->buf; - if (!fw_buf) - goto out; - - switch (loading) { - case 1: - /* discarding any previous partial load */ - if (!test_bit(FW_STATUS_DONE, &fw_buf->status)) { - for (i = 0; i < fw_buf->nr_pages; i++) - __free_page(fw_buf->pages[i]); - kfree(fw_buf->pages); - fw_buf->pages = NULL; - fw_buf->page_array_size = 0; - fw_buf->nr_pages = 0; - set_bit(FW_STATUS_LOADING, &fw_buf->status); - } - break; - case 0: - if (test_bit(FW_STATUS_LOADING, &fw_buf->status)) { - set_bit(FW_STATUS_DONE, &fw_buf->status); - clear_bit(FW_STATUS_LOADING, &fw_buf->status); - - /* - * Several loading requests may be pending on - * one same firmware buf, so let all requests - * see the mapped 'buf->data' once the loading - * is completed. - * */ - fw_map_pages_buf(fw_buf); - list_del_init(&fw_buf->pending_list); - complete_all(&fw_buf->completion); - break; - } - /* fallthrough */ - default: - dev_err(dev, "%s: unexpected value (%d)\n", __func__, loading); - /* fallthrough */ - case -1: - fw_load_abort(fw_priv); - break; - } -out: - mutex_unlock(&fw_lock); - return count; -} - -static DEVICE_ATTR(loading, 0644, firmware_loading_show, firmware_loading_store); - -static ssize_t firmware_data_read(struct file *filp, struct kobject *kobj, - struct bin_attribute *bin_attr, - char *buffer, loff_t offset, size_t count) -{ - struct device *dev = kobj_to_dev(kobj); - struct firmware_priv *fw_priv = to_firmware_priv(dev); - struct firmware_buf *buf; - ssize_t ret_count; - - mutex_lock(&fw_lock); - buf = fw_priv->buf; - if (!buf || test_bit(FW_STATUS_DONE, &buf->status)) { - ret_count = -ENODEV; - goto out; - } - if (offset > buf->size) { - ret_count = 0; - goto out; - } - if (count > buf->size - offset) - count = buf->size - offset; - - ret_count = count; - - while (count) { - void *page_data; - int page_nr = offset >> PAGE_SHIFT; - int page_ofs = offset & (PAGE_SIZE-1); - int page_cnt = min_t(size_t, PAGE_SIZE - page_ofs, count); - - page_data = kmap(buf->pages[page_nr]); - - memcpy(buffer, page_data + page_ofs, page_cnt); - - kunmap(buf->pages[page_nr]); - buffer += page_cnt; - offset += page_cnt; - count -= page_cnt; - } -out: - mutex_unlock(&fw_lock); - return ret_count; -} - -static int fw_realloc_buffer(struct firmware_priv *fw_priv, int min_size) -{ - struct firmware_buf *buf = fw_priv->buf; - int pages_needed = ALIGN(min_size, PAGE_SIZE) >> PAGE_SHIFT; - - /* If the array of pages is too small, grow it... */ - if (buf->page_array_size < pages_needed) { - int new_array_size = max(pages_needed, - buf->page_array_size * 2); - struct page **new_pages; - - new_pages = kmalloc(new_array_size * sizeof(void *), - GFP_KERNEL); - if (!new_pages) { - fw_load_abort(fw_priv); - return -ENOMEM; - } - memcpy(new_pages, buf->pages, - buf->page_array_size * sizeof(void *)); - memset(&new_pages[buf->page_array_size], 0, sizeof(void *) * - (new_array_size - buf->page_array_size)); - kfree(buf->pages); - buf->pages = new_pages; - buf->page_array_size = new_array_size; - } - - while (buf->nr_pages < pages_needed) { - buf->pages[buf->nr_pages] = - alloc_page(GFP_KERNEL | __GFP_HIGHMEM); - - if (!buf->pages[buf->nr_pages]) { - fw_load_abort(fw_priv); - return -ENOMEM; - } - buf->nr_pages++; - } - return 0; -} - -/** - * firmware_data_write - write method for firmware - * @filp: open sysfs file - * @kobj: kobject for the device - * @bin_attr: bin_attr structure - * @buffer: buffer being written - * @offset: buffer offset for write in total data store area - * @count: buffer size - * - * Data written to the 'data' attribute will be later handed to - * the driver as a firmware image. - **/ -static ssize_t firmware_data_write(struct file *filp, struct kobject *kobj, - struct bin_attribute *bin_attr, - char *buffer, loff_t offset, size_t count) -{ - struct device *dev = kobj_to_dev(kobj); - struct firmware_priv *fw_priv = to_firmware_priv(dev); - struct firmware_buf *buf; - ssize_t retval; - - if (!capable(CAP_SYS_RAWIO)) - return -EPERM; - - mutex_lock(&fw_lock); - buf = fw_priv->buf; - if (!buf || test_bit(FW_STATUS_DONE, &buf->status)) { - retval = -ENODEV; - goto out; - } - - retval = fw_realloc_buffer(fw_priv, offset + count); - if (retval) - goto out; - - retval = count; - - while (count) { - void *page_data; - int page_nr = offset >> PAGE_SHIFT; - int page_ofs = offset & (PAGE_SIZE - 1); - int page_cnt = min_t(size_t, PAGE_SIZE - page_ofs, count); - - page_data = kmap(buf->pages[page_nr]); - - memcpy(page_data + page_ofs, buffer, page_cnt); - - kunmap(buf->pages[page_nr]); - buffer += page_cnt; - offset += page_cnt; - count -= page_cnt; - } - - buf->size = max_t(size_t, offset, buf->size); -out: - mutex_unlock(&fw_lock); - return retval; -} - -static struct bin_attribute firmware_attr_data = { - .attr = { .name = "data", .mode = 0644 }, - .size = 0, - .read = firmware_data_read, - .write = firmware_data_write, -}; - -static void firmware_class_timeout_work(struct work_struct *work) -{ - struct firmware_priv *fw_priv = container_of(work, - struct firmware_priv, timeout_work.work); - - mutex_lock(&fw_lock); - fw_load_abort(fw_priv); - mutex_unlock(&fw_lock); -} - -static struct firmware_priv * -fw_create_instance(struct firmware *firmware, const char *fw_name, - struct device *device, bool uevent, bool nowait) -{ - struct firmware_priv *fw_priv; - struct device *f_dev; - - fw_priv = kzalloc(sizeof(*fw_priv), GFP_KERNEL); - if (!fw_priv) { - dev_err(device, "%s: kmalloc failed\n", __func__); - fw_priv = ERR_PTR(-ENOMEM); - goto exit; - } - - fw_priv->nowait = nowait; - fw_priv->fw = firmware; - INIT_DELAYED_WORK(&fw_priv->timeout_work, - firmware_class_timeout_work); - - f_dev = &fw_priv->dev; - - device_initialize(f_dev); - dev_set_name(f_dev, "%s", fw_name); - f_dev->parent = device; - f_dev->class = &firmware_class; -exit: - return fw_priv; -} - -/* load a firmware via user helper */ -static int _request_firmware_load(struct firmware_priv *fw_priv, bool uevent, - long timeout) -{ - int retval = 0; - struct device *f_dev = &fw_priv->dev; - struct firmware_buf *buf = fw_priv->buf; - - /* fall back on userspace loading */ - buf->is_paged_buf = true; - - dev_set_uevent_suppress(f_dev, true); - - retval = device_add(f_dev); - if (retval) { - dev_err(f_dev, "%s: device_register failed\n", __func__); - goto err_put_dev; - } - - retval = device_create_bin_file(f_dev, &firmware_attr_data); - if (retval) { - dev_err(f_dev, "%s: sysfs_create_bin_file failed\n", __func__); - goto err_del_dev; - } - - retval = device_create_file(f_dev, &dev_attr_loading); - if (retval) { - dev_err(f_dev, "%s: device_create_file failed\n", __func__); - goto err_del_bin_attr; - } - - if (uevent) { - buf->need_uevent = true; - dev_set_uevent_suppress(f_dev, false); - dev_dbg(f_dev, "firmware: requesting %s\n", buf->fw_id); - if (timeout != MAX_SCHEDULE_TIMEOUT) - schedule_delayed_work(&fw_priv->timeout_work, timeout); - - kobject_uevent(&fw_priv->dev.kobj, KOBJ_ADD); - } - - mutex_lock(&fw_lock); - list_add(&buf->pending_list, &pending_fw_head); - mutex_unlock(&fw_lock); - - wait_for_completion(&buf->completion); - - cancel_delayed_work_sync(&fw_priv->timeout_work); - - device_remove_file(f_dev, &dev_attr_loading); -err_del_bin_attr: - device_remove_bin_file(f_dev, &firmware_attr_data); -err_del_dev: - device_del(f_dev); -err_put_dev: - put_device(f_dev); - return retval; -} - -static int fw_load_from_user_helper(struct firmware *firmware, - const char *name, struct device *device, - bool uevent, bool nowait, long timeout) -{ - struct firmware_priv *fw_priv; - - fw_priv = fw_create_instance(firmware, name, device, uevent, nowait); - if (IS_ERR(fw_priv)) - return PTR_ERR(fw_priv); - - fw_priv->buf = firmware->priv; - return _request_firmware_load(fw_priv, uevent, timeout); -} - -#ifdef CONFIG_PM_SLEEP -/* kill pending requests without uevent to avoid blocking suspend */ -static void kill_requests_without_uevent(void) -{ - struct firmware_buf *buf; - struct firmware_buf *next; - - mutex_lock(&fw_lock); - list_for_each_entry_safe(buf, next, &pending_fw_head, pending_list) { - if (!buf->need_uevent) - __fw_load_abort(buf); - } - mutex_unlock(&fw_lock); -} -#endif - -#else /* CONFIG_FW_LOADER_USER_HELPER */ -static inline int -fw_load_from_user_helper(struct firmware *firmware, const char *name, - struct device *device, bool uevent, bool nowait, - long timeout) -{ - return -ENOENT; -} - -/* No abort during direct loading */ -#define is_fw_load_aborted(buf) false - -#ifdef CONFIG_PM_SLEEP -static inline void kill_requests_without_uevent(void) { } -#endif - -#endif /* CONFIG_FW_LOADER_USER_HELPER */ - - -/* wait until the shared firmware_buf becomes ready (or error) */ -static int sync_cached_firmware_buf(struct firmware_buf *buf) -{ - int ret = 0; - - mutex_lock(&fw_lock); - while (!test_bit(FW_STATUS_DONE, &buf->status)) { - if (is_fw_load_aborted(buf)) { - ret = -ENOENT; - break; - } - mutex_unlock(&fw_lock); - wait_for_completion(&buf->completion); - mutex_lock(&fw_lock); - } - mutex_unlock(&fw_lock); - return ret; -} - -/* prepare firmware and firmware_buf structs; - * return 0 if a firmware is already assigned, 1 if need to load one, - * or a negative error code - */ -static int -_request_firmware_prepare(struct firmware **firmware_p, const char *name, - struct device *device) -{ - struct firmware *firmware; - struct firmware_buf *buf; - int ret; - - *firmware_p = firmware = kzalloc(sizeof(*firmware), GFP_KERNEL); - if (!firmware) { - dev_err(device, "%s: kmalloc(struct firmware) failed\n", - __func__); - return -ENOMEM; - } - - if (fw_get_builtin_firmware(firmware, name)) { - dev_dbg(device, "firmware: using built-in firmware %s\n", name); - return 0; /* assigned */ - } - - ret = fw_lookup_and_allocate_buf(name, &fw_cache, &buf); - - /* - * bind with 'buf' now to avoid warning in failure path - * of requesting firmware. - */ - firmware->priv = buf; - - if (ret > 0) { - ret = sync_cached_firmware_buf(buf); - if (!ret) { - fw_set_page_data(buf, firmware); - return 0; /* assigned */ - } - } - - if (ret < 0) - return ret; - return 1; /* need to load */ -} - -static int assign_firmware_buf(struct firmware *fw, struct device *device, - bool skip_cache) -{ - struct firmware_buf *buf = fw->priv; - - mutex_lock(&fw_lock); - if (!buf->size || is_fw_load_aborted(buf)) { - mutex_unlock(&fw_lock); - return -ENOENT; - } - - /* - * add firmware name into devres list so that we can auto cache - * and uncache firmware for device. - * - * device may has been deleted already, but the problem - * should be fixed in devres or driver core. - */ - if (device && !skip_cache) - fw_add_devm_name(device, buf->fw_id); - - /* - * After caching firmware image is started, let it piggyback - * on request firmware. - */ - if (buf->fwc->state == FW_LOADER_START_CACHE) { - if (fw_cache_piggyback_on_request(buf->fw_id)) - kref_get(&buf->ref); - } - - /* pass the pages buffer to driver at the last minute */ - fw_set_page_data(buf, fw); - mutex_unlock(&fw_lock); - return 0; -} - -/* called from request_firmware() and request_firmware_work_func() */ -static int -_request_firmware(const struct firmware **firmware_p, const char *name, - struct device *device, bool uevent, bool nowait) -{ - struct firmware *fw; - long timeout; - int ret; - - if (!firmware_p) - return -EINVAL; - - ret = _request_firmware_prepare(&fw, name, device); - if (ret <= 0) /* error or already assigned */ - goto out; - - ret = 0; - timeout = firmware_loading_timeout(); - if (nowait) { - timeout = usermodehelper_read_lock_wait(timeout); - if (!timeout) { - dev_dbg(device, "firmware: %s loading timed out\n", - name); - ret = -EBUSY; - goto out; - } - } else { - ret = usermodehelper_read_trylock(); - if (WARN_ON(ret)) { - dev_err(device, "firmware: %s will not be loaded\n", - name); - goto out; - } - } - - if (!fw_get_filesystem_firmware(device, fw->priv)) - ret = fw_load_from_user_helper(fw, name, device, - uevent, nowait, timeout); - - /* don't cache firmware handled without uevent */ - if (!ret) - ret = assign_firmware_buf(fw, device, !uevent); - - usermodehelper_read_unlock(); - - out: - if (ret < 0) { - release_firmware(fw); - fw = NULL; - } - - *firmware_p = fw; - return ret; -} - -/** - * request_firmware: - send firmware request and wait for it - * @firmware_p: pointer to firmware image - * @name: name of firmware file - * @device: device for which firmware is being loaded - * - * @firmware_p will be used to return a firmware image by the name - * of @name for device @device. - * - * Should be called from user context where sleeping is allowed. - * - * @name will be used as $FIRMWARE in the uevent environment and - * should be distinctive enough not to be confused with any other - * firmware image for this or any other device. - * - * Caller must hold the reference count of @device. - * - * The function can be called safely inside device's suspend and - * resume callback. - **/ -int -request_firmware(const struct firmware **firmware_p, const char *name, - struct device *device) -{ - int ret; - - /* Need to pin this module until return */ - __module_get(THIS_MODULE); - ret = _request_firmware(firmware_p, name, device, true, false); - module_put(THIS_MODULE); - return ret; -} -EXPORT_SYMBOL(request_firmware); - -/** - * release_firmware: - release the resource associated with a firmware image - * @fw: firmware resource to release - **/ -void release_firmware(const struct firmware *fw) -{ - if (fw) { - if (!fw_is_builtin_firmware(fw)) - firmware_free_data(fw); - kfree(fw); - } -} -EXPORT_SYMBOL(release_firmware); - -/* Async support */ -struct firmware_work { - struct work_struct work; - struct module *module; - const char *name; - struct device *device; - void *context; - void (*cont)(const struct firmware *fw, void *context); - bool uevent; -}; - -static void request_firmware_work_func(struct work_struct *work) -{ - struct firmware_work *fw_work; - const struct firmware *fw; - - fw_work = container_of(work, struct firmware_work, work); - - _request_firmware(&fw, fw_work->name, fw_work->device, - fw_work->uevent, true); - fw_work->cont(fw, fw_work->context); - put_device(fw_work->device); /* taken in request_firmware_nowait() */ - - module_put(fw_work->module); - kfree(fw_work); -} - -/** - * request_firmware_nowait - asynchronous version of request_firmware - * @module: module requesting the firmware - * @uevent: sends uevent to copy the firmware image if this flag - * is non-zero else the firmware copy must be done manually. - * @name: name of firmware file - * @device: device for which firmware is being loaded - * @gfp: allocation flags - * @context: will be passed over to @cont, and - * @fw may be %NULL if firmware request fails. - * @cont: function will be called asynchronously when the firmware - * request is over. - * - * Caller must hold the reference count of @device. - * - * Asynchronous variant of request_firmware() for user contexts: - * - sleep for as small periods as possible since it may - * increase kernel boot time of built-in device drivers - * requesting firmware in their ->probe() methods, if - * @gfp is GFP_KERNEL. - * - * - can't sleep at all if @gfp is GFP_ATOMIC. - **/ -int -request_firmware_nowait( - struct module *module, bool uevent, - const char *name, struct device *device, gfp_t gfp, void *context, - void (*cont)(const struct firmware *fw, void *context)) -{ - struct firmware_work *fw_work; - - fw_work = kzalloc(sizeof (struct firmware_work), gfp); - if (!fw_work) - return -ENOMEM; - - fw_work->module = module; - fw_work->name = name; - fw_work->device = device; - fw_work->context = context; - fw_work->cont = cont; - fw_work->uevent = uevent; - - if (!try_module_get(module)) { - kfree(fw_work); - return -EFAULT; - } - - get_device(fw_work->device); - INIT_WORK(&fw_work->work, request_firmware_work_func); - schedule_work(&fw_work->work); - return 0; -} -EXPORT_SYMBOL(request_firmware_nowait); - -#ifdef CONFIG_PM_SLEEP -static ASYNC_DOMAIN_EXCLUSIVE(fw_cache_domain); - -/** - * cache_firmware - cache one firmware image in kernel memory space - * @fw_name: the firmware image name - * - * Cache firmware in kernel memory so that drivers can use it when - * system isn't ready for them to request firmware image from userspace. - * Once it returns successfully, driver can use request_firmware or its - * nowait version to get the cached firmware without any interacting - * with userspace - * - * Return 0 if the firmware image has been cached successfully - * Return !0 otherwise - * - */ -static int cache_firmware(const char *fw_name) -{ - int ret; - const struct firmware *fw; - - pr_debug("%s: %s\n", __func__, fw_name); - - ret = request_firmware(&fw, fw_name, NULL); - if (!ret) - kfree(fw); - - pr_debug("%s: %s ret=%d\n", __func__, fw_name, ret); - - return ret; -} - -static struct firmware_buf *fw_lookup_buf(const char *fw_name) -{ - struct firmware_buf *tmp; - struct firmware_cache *fwc = &fw_cache; - - spin_lock(&fwc->lock); - tmp = __fw_lookup_buf(fw_name); - spin_unlock(&fwc->lock); - - return tmp; -} - -/** - * uncache_firmware - remove one cached firmware image - * @fw_name: the firmware image name - * - * Uncache one firmware image which has been cached successfully - * before. - * - * Return 0 if the firmware cache has been removed successfully - * Return !0 otherwise - * - */ -static int uncache_firmware(const char *fw_name) -{ - struct firmware_buf *buf; - struct firmware fw; - - pr_debug("%s: %s\n", __func__, fw_name); - - if (fw_get_builtin_firmware(&fw, fw_name)) - return 0; - - buf = fw_lookup_buf(fw_name); - if (buf) { - fw_free_buf(buf); - return 0; - } - - return -EINVAL; -} - -static struct fw_cache_entry *alloc_fw_cache_entry(const char *name) -{ - struct fw_cache_entry *fce; - - fce = kzalloc(sizeof(*fce) + strlen(name) + 1, GFP_ATOMIC); - if (!fce) - goto exit; - - strcpy(fce->name, name); -exit: - return fce; -} - -static int __fw_entry_found(const char *name) -{ - struct firmware_cache *fwc = &fw_cache; - struct fw_cache_entry *fce; - - list_for_each_entry(fce, &fwc->fw_names, list) { - if (!strcmp(fce->name, name)) - return 1; - } - return 0; -} - -static int fw_cache_piggyback_on_request(const char *name) -{ - struct firmware_cache *fwc = &fw_cache; - struct fw_cache_entry *fce; - int ret = 0; - - spin_lock(&fwc->name_lock); - if (__fw_entry_found(name)) - goto found; - - fce = alloc_fw_cache_entry(name); - if (fce) { - ret = 1; - list_add(&fce->list, &fwc->fw_names); - pr_debug("%s: fw: %s\n", __func__, name); - } -found: - spin_unlock(&fwc->name_lock); - return ret; -} - -static void free_fw_cache_entry(struct fw_cache_entry *fce) -{ - kfree(fce); -} - -static void __async_dev_cache_fw_image(void *fw_entry, - async_cookie_t cookie) -{ - struct fw_cache_entry *fce = fw_entry; - struct firmware_cache *fwc = &fw_cache; - int ret; - - ret = cache_firmware(fce->name); - if (ret) { - spin_lock(&fwc->name_lock); - list_del(&fce->list); - spin_unlock(&fwc->name_lock); - - free_fw_cache_entry(fce); - } -} - -/* called with dev->devres_lock held */ -static void dev_create_fw_entry(struct device *dev, void *res, - void *data) -{ - struct fw_name_devm *fwn = res; - const char *fw_name = fwn->name; - struct list_head *head = data; - struct fw_cache_entry *fce; - - fce = alloc_fw_cache_entry(fw_name); - if (fce) - list_add(&fce->list, head); -} - -static int devm_name_match(struct device *dev, void *res, - void *match_data) -{ - struct fw_name_devm *fwn = res; - return (fwn->magic == (unsigned long)match_data); -} - -static void dev_cache_fw_image(struct device *dev, void *data) -{ - LIST_HEAD(todo); - struct fw_cache_entry *fce; - struct fw_cache_entry *fce_next; - struct firmware_cache *fwc = &fw_cache; - - devres_for_each_res(dev, fw_name_devm_release, - devm_name_match, &fw_cache, - dev_create_fw_entry, &todo); - - list_for_each_entry_safe(fce, fce_next, &todo, list) { - list_del(&fce->list); - - spin_lock(&fwc->name_lock); - /* only one cache entry for one firmware */ - if (!__fw_entry_found(fce->name)) { - list_add(&fce->list, &fwc->fw_names); - } else { - free_fw_cache_entry(fce); - fce = NULL; - } - spin_unlock(&fwc->name_lock); - - if (fce) - async_schedule_domain(__async_dev_cache_fw_image, - (void *)fce, - &fw_cache_domain); - } -} - -static void __device_uncache_fw_images(void) -{ - struct firmware_cache *fwc = &fw_cache; - struct fw_cache_entry *fce; - - spin_lock(&fwc->name_lock); - while (!list_empty(&fwc->fw_names)) { - fce = list_entry(fwc->fw_names.next, - struct fw_cache_entry, list); - list_del(&fce->list); - spin_unlock(&fwc->name_lock); - - uncache_firmware(fce->name); - free_fw_cache_entry(fce); - - spin_lock(&fwc->name_lock); - } - spin_unlock(&fwc->name_lock); -} - -/** - * device_cache_fw_images - cache devices' firmware - * - * If one device called request_firmware or its nowait version - * successfully before, the firmware names are recored into the - * device's devres link list, so device_cache_fw_images can call - * cache_firmware() to cache these firmwares for the device, - * then the device driver can load its firmwares easily at - * time when system is not ready to complete loading firmware. - */ -static void device_cache_fw_images(void) -{ - struct firmware_cache *fwc = &fw_cache; - int old_timeout; - DEFINE_WAIT(wait); - - pr_debug("%s\n", __func__); - - /* cancel uncache work */ - cancel_delayed_work_sync(&fwc->work); - - /* - * use small loading timeout for caching devices' firmware - * because all these firmware images have been loaded - * successfully at lease once, also system is ready for - * completing firmware loading now. The maximum size of - * firmware in current distributions is about 2M bytes, - * so 10 secs should be enough. - */ - old_timeout = loading_timeout; - loading_timeout = 10; - - mutex_lock(&fw_lock); - fwc->state = FW_LOADER_START_CACHE; - dpm_for_each_dev(NULL, dev_cache_fw_image); - mutex_unlock(&fw_lock); - - /* wait for completion of caching firmware for all devices */ - async_synchronize_full_domain(&fw_cache_domain); - - loading_timeout = old_timeout; -} - -/** - * device_uncache_fw_images - uncache devices' firmware - * - * uncache all firmwares which have been cached successfully - * by device_uncache_fw_images earlier - */ -static void device_uncache_fw_images(void) -{ - pr_debug("%s\n", __func__); - __device_uncache_fw_images(); -} - -static void device_uncache_fw_images_work(struct work_struct *work) -{ - device_uncache_fw_images(); -} - -/** - * device_uncache_fw_images_delay - uncache devices firmwares - * @delay: number of milliseconds to delay uncache device firmwares - * - * uncache all devices's firmwares which has been cached successfully - * by device_cache_fw_images after @delay milliseconds. - */ -static void device_uncache_fw_images_delay(unsigned long delay) -{ - schedule_delayed_work(&fw_cache.work, - msecs_to_jiffies(delay)); -} - -static int fw_pm_notify(struct notifier_block *notify_block, - unsigned long mode, void *unused) -{ - switch (mode) { - case PM_HIBERNATION_PREPARE: - case PM_SUSPEND_PREPARE: - kill_requests_without_uevent(); - device_cache_fw_images(); - break; - - case PM_POST_SUSPEND: - case PM_POST_HIBERNATION: - case PM_POST_RESTORE: - /* - * In case that system sleep failed and syscore_suspend is - * not called. - */ - mutex_lock(&fw_lock); - fw_cache.state = FW_LOADER_NO_CACHE; - mutex_unlock(&fw_lock); - - device_uncache_fw_images_delay(10 * MSEC_PER_SEC); - break; - } - - return 0; -} - -/* stop caching firmware once syscore_suspend is reached */ -static int fw_suspend(void) -{ - fw_cache.state = FW_LOADER_NO_CACHE; - return 0; -} - -static struct syscore_ops fw_syscore_ops = { - .suspend = fw_suspend, -}; -#else -static int fw_cache_piggyback_on_request(const char *name) -{ - return 0; -} -#endif - -static void __init fw_cache_init(void) -{ - spin_lock_init(&fw_cache.lock); - INIT_LIST_HEAD(&fw_cache.head); - fw_cache.state = FW_LOADER_NO_CACHE; - -#ifdef CONFIG_PM_SLEEP - spin_lock_init(&fw_cache.name_lock); - INIT_LIST_HEAD(&fw_cache.fw_names); - - INIT_DELAYED_WORK(&fw_cache.work, - device_uncache_fw_images_work); - - fw_cache.pm_notify.notifier_call = fw_pm_notify; - register_pm_notifier(&fw_cache.pm_notify); - - register_syscore_ops(&fw_syscore_ops); -#endif -} - -static int __init firmware_class_init(void) -{ - fw_cache_init(); -#ifdef CONFIG_FW_LOADER_USER_HELPER - register_reboot_notifier(&fw_shutdown_nb); - return class_register(&firmware_class); -#else - return 0; -#endif -} - -static void __exit firmware_class_exit(void) -{ -#ifdef CONFIG_PM_SLEEP - unregister_syscore_ops(&fw_syscore_ops); - unregister_pm_notifier(&fw_cache.pm_notify); -#endif -#ifdef CONFIG_FW_LOADER_USER_HELPER - unregister_reboot_notifier(&fw_shutdown_nb); - class_unregister(&firmware_class); -#endif -} - -fs_initcall(firmware_class_init); -module_exit(firmware_class_exit); diff --git a/drivers/base/firmware_loader/Kconfig b/drivers/base/firmware_loader/Kconfig new file mode 100644 index 000000000000..15eff8a4b505 --- /dev/null +++ b/drivers/base/firmware_loader/Kconfig @@ -0,0 +1,239 @@ +# SPDX-License-Identifier: GPL-2.0 +menu "Firmware loader" + +config FW_LOADER + tristate "Firmware loading facility" if EXPERT + select CRYPTO_LIB_SHA256 if FW_LOADER_DEBUG + default y + help + This enables the firmware loading facility in the kernel. The kernel + will first look for built-in firmware, if it has any. Next, it will + look for the requested firmware in a series of filesystem paths: + + o firmware_class path module parameter or kernel boot param + o /lib/firmware/updates/UTS_RELEASE + o /lib/firmware/updates + o /lib/firmware/UTS_RELEASE + o /lib/firmware + + Enabling this feature only increases your kernel image by about + 828 bytes, enable this option unless you are certain you don't + need firmware. + + You typically want this built-in (=y) but you can also enable this + as a module, in which case the firmware_class module will be built. + You also want to be sure to enable this built-in if you are going to + enable built-in firmware (CONFIG_EXTRA_FIRMWARE). + +config FW_LOADER_DEBUG + bool "Log filenames and checksums for loaded firmware" + depends on DYNAMIC_DEBUG + depends on FW_LOADER + default FW_LOADER + help + Select this option to use dynamic debug to log firmware filenames and + SHA256 checksums to the kernel log for each firmware file that is + loaded. + +config RUST_FW_LOADER_ABSTRACTIONS + bool "Rust Firmware Loader abstractions" + depends on RUST + select FW_LOADER + help + This enables the Rust abstractions for the firmware loader API. + +if FW_LOADER + +config FW_LOADER_PAGED_BUF + bool + +config FW_LOADER_SYSFS + bool + +config EXTRA_FIRMWARE + string "Build named firmware blobs into the kernel binary" + help + Device drivers which require firmware can typically deal with + having the kernel load firmware from the various supported + /lib/firmware/ paths. This option enables you to build into the + kernel firmware files. Built-in firmware searches are preceded + over firmware lookups using your filesystem over the supported + /lib/firmware paths documented on CONFIG_FW_LOADER. + + This may be useful for testing or if the firmware is required early on + in boot and cannot rely on the firmware being placed in an initrd or + initramfs. + + This option is a string and takes the (space-separated) names of the + firmware files -- the same names that appear in MODULE_FIRMWARE() + and request_firmware() in the source. These files should exist under + the directory specified by the EXTRA_FIRMWARE_DIR option, which is + /lib/firmware by default. + + For example, you might set CONFIG_EXTRA_FIRMWARE="usb8388.bin", copy + the usb8388.bin file into /lib/firmware, and build the kernel. Then + any request_firmware("usb8388.bin") will be satisfied internally + inside the kernel without ever looking at your filesystem at runtime. + + WARNING: If you include additional firmware files into your binary + kernel image that are not available under the terms of the GPL, + then it may be a violation of the GPL to distribute the resulting + image since it combines both GPL and non-GPL work. You should + consult a lawyer of your own before distributing such an image. + + NOTE: Compressed files are not supported in EXTRA_FIRMWARE. + +config EXTRA_FIRMWARE_DIR + string "Firmware blobs root directory" + depends on EXTRA_FIRMWARE != "" + default "/lib/firmware" + help + This option controls the directory in which the kernel build system + looks for the firmware files listed in the EXTRA_FIRMWARE option. + +config FW_LOADER_USER_HELPER + bool "Enable the firmware sysfs fallback mechanism" + select FW_LOADER_SYSFS + select FW_LOADER_PAGED_BUF + help + This option enables a sysfs loading facility to enable firmware + loading to the kernel through userspace as a fallback mechanism + if and only if the kernel's direct filesystem lookup for the + firmware failed using the different /lib/firmware/ paths, or the + path specified in the firmware_class path module parameter, or the + firmware_class path kernel boot parameter if the firmware_class is + built-in. For details on how to work with the sysfs fallback mechanism + refer to Documentation/driver-api/firmware/fallback-mechanisms.rst. + + The direct filesystem lookup for firmware is always used first now. + + If the kernel's direct filesystem lookup for firmware fails to find + the requested firmware a sysfs fallback loading facility is made + available and userspace is informed about this through uevents. + The uevent can be suppressed if the driver explicitly requested it, + this is known as the driver using the custom fallback mechanism. + If the custom fallback mechanism is used userspace must always + acknowledge failure to find firmware as the timeout for the fallback + mechanism is disabled, and failed requests will linger forever. + + This used to be the default firmware loading facility, and udev used + to listen for uvents to load firmware for the kernel. The firmware + loading facility functionality in udev has been removed, as such it + can no longer be relied upon as a fallback mechanism. Linux no longer + relies on or uses a fallback mechanism in userspace. If you need to + rely on one refer to the permissively licensed firmwared: + + https://github.com/teg/firmwared + + Since this was the default firmware loading facility at one point, + old userspace may exist which relies upon it, and as such this + mechanism can never be removed from the kernel. + + You should only enable this functionality if you are certain you + require a fallback mechanism and have a userspace mechanism ready to + load firmware in case it is not found. One main reason for this may + be if you have drivers which require firmware built-in and for + whatever reason cannot place the required firmware in initramfs. + Another reason kernels may have this feature enabled is to support a + driver which explicitly relies on this fallback mechanism. Only two + drivers need this today: + + o CONFIG_LEDS_LP55XX_COMMON + o CONFIG_DELL_RBU + + Outside of supporting the above drivers, another reason for needing + this may be that your firmware resides outside of the paths the kernel + looks for and cannot possibly be specified using the firmware_class + path module parameter or kernel firmware_class path boot parameter + if firmware_class is built-in. + + A modern use case may be to temporarily mount a custom partition + during provisioning which is only accessible to userspace, and then + to use it to look for and fetch the required firmware. Such type of + driver functionality may not even ever be desirable upstream by + vendors, and as such is only required to be supported as an interface + for provisioning. Since udev's firmware loading facility has been + removed you can use firmwared or a fork of it to customize how you + want to load firmware based on uevents issued. + + Enabling this option will increase your kernel image size by about + 13436 bytes. + + If you are unsure about this, say N here, unless you are Linux + distribution and need to support the above two drivers, or you are + certain you need to support some really custom firmware loading + facility in userspace. + +config FW_LOADER_USER_HELPER_FALLBACK + bool "Force the firmware sysfs fallback mechanism when possible" + depends on FW_LOADER_USER_HELPER + help + Enabling this option forces a sysfs userspace fallback mechanism + to be used for all firmware requests which explicitly do not disable a + a fallback mechanism. Firmware calls which do prohibit a fallback + mechanism is request_firmware_direct(). This option is kept for + backward compatibility purposes given this precise mechanism can also + be enabled by setting the proc sysctl value to true: + + /proc/sys/kernel/firmware_config/force_sysfs_fallback + + If you are unsure about this, say N here. + +config FW_LOADER_COMPRESS + bool "Enable compressed firmware support" + help + This option enables the support for loading compressed firmware + files. The caller of firmware API receives the decompressed file + content. The compressed file is loaded as a fallback, only after + loading the raw file failed at first. + + Compressed firmware support does not apply to firmware images + that are built into the kernel image (CONFIG_EXTRA_FIRMWARE). + +if FW_LOADER_COMPRESS +config FW_LOADER_COMPRESS_XZ + bool "Enable XZ-compressed firmware support" + select FW_LOADER_PAGED_BUF + select XZ_DEC + default y + help + This option adds the support for XZ-compressed files. + The files have to be compressed with either none or crc32 + integrity check type (pass "-C crc32" option to xz command). + +config FW_LOADER_COMPRESS_ZSTD + bool "Enable ZSTD-compressed firmware support" + select ZSTD_DECOMPRESS + help + This option adds the support for ZSTD-compressed files. + +endif # FW_LOADER_COMPRESS + +config FW_CACHE + bool "Enable firmware caching during suspend" + depends on PM_SLEEP + default y if PM_SLEEP + help + Because firmware caching generates uevent messages that are sent + over a netlink socket, it can prevent suspend on many platforms. + It is also not always useful, so on such platforms we have the + option. + + If unsure, say Y. + +config FW_UPLOAD + bool "Enable users to initiate firmware updates using sysfs" + select FW_LOADER_SYSFS + select FW_LOADER_PAGED_BUF + help + Enabling this option will allow device drivers to expose a persistent + sysfs interface that allows firmware updates to be initiated from + userspace. For example, FPGA based PCIe cards load firmware and FPGA + images from local FLASH when the card boots. The images in FLASH may + be updated with new images provided by the user. Enable this device + to support cards that rely on user-initiated updates for firmware files. + + If unsure, say N. + +endif # FW_LOADER +endmenu diff --git a/drivers/base/firmware_loader/Makefile b/drivers/base/firmware_loader/Makefile new file mode 100644 index 000000000000..60d19f9e0ddc --- /dev/null +++ b/drivers/base/firmware_loader/Makefile @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: GPL-2.0 +# Makefile for the Linux firmware loader + +obj-$(CONFIG_FW_LOADER_USER_HELPER) += fallback_table.o +obj-$(CONFIG_FW_LOADER) += firmware_class.o +firmware_class-objs := main.o +firmware_class-$(CONFIG_FW_LOADER_USER_HELPER) += fallback.o +firmware_class-$(CONFIG_EFI_EMBEDDED_FIRMWARE) += fallback_platform.o +firmware_class-$(CONFIG_FW_LOADER_SYSFS) += sysfs.o +firmware_class-$(CONFIG_FW_UPLOAD) += sysfs_upload.o + +obj-y += builtin/ diff --git a/drivers/base/firmware_loader/builtin/.gitignore b/drivers/base/firmware_loader/builtin/.gitignore new file mode 100644 index 000000000000..166f76b43049 --- /dev/null +++ b/drivers/base/firmware_loader/builtin/.gitignore @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0 +*.gen.S diff --git a/drivers/base/firmware_loader/builtin/Makefile b/drivers/base/firmware_loader/builtin/Makefile new file mode 100644 index 000000000000..6c067dedc01e --- /dev/null +++ b/drivers/base/firmware_loader/builtin/Makefile @@ -0,0 +1,42 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-y += main.o + +# Create $(fwdir) from $(CONFIG_EXTRA_FIRMWARE_DIR) -- if it doesn't have a +# leading /, it's relative to $(srctree). +fwdir := $(CONFIG_EXTRA_FIRMWARE_DIR) +fwdir := $(addprefix $(srctree)/,$(filter-out /%,$(fwdir)))$(filter /%,$(fwdir)) + +firmware := $(addsuffix .gen.o, $(CONFIG_EXTRA_FIRMWARE)) +obj-y += $(firmware) + +FWNAME = $(patsubst $(obj)/%.gen.S,%,$@) +FWSTR = $(subst $(comma),_,$(subst /,_,$(subst .,_,$(subst -,_,$(FWNAME))))) +ASM_WORD = $(if $(CONFIG_64BIT),.quad,.long) +ASM_ALIGN = $(if $(CONFIG_64BIT),3,2) +PROGBITS = $(if $(CONFIG_ARM),%,@)progbits + +filechk_fwbin = \ + echo "/* Generated by $(src)/Makefile */" ;\ + echo " .section .rodata" ;\ + echo " .p2align 4" ;\ + echo "_fw_$(FWSTR)_bin:" ;\ + echo " .incbin \"$(fwdir)/$(FWNAME)\"" ;\ + echo "_fw_end:" ;\ + echo " .section .rodata.str,\"aMS\",$(PROGBITS),1" ;\ + echo " .p2align $(ASM_ALIGN)" ;\ + echo "_fw_$(FWSTR)_name:" ;\ + echo " .string \"$(FWNAME)\"" ;\ + echo " .section .builtin_fw,\"a\",$(PROGBITS)" ;\ + echo " .p2align $(ASM_ALIGN)" ;\ + echo " $(ASM_WORD) _fw_$(FWSTR)_name" ;\ + echo " $(ASM_WORD) _fw_$(FWSTR)_bin" ;\ + echo " $(ASM_WORD) _fw_end - _fw_$(FWSTR)_bin" + +$(obj)/%.gen.S: FORCE + $(call filechk,fwbin) + +# The .o files depend on the binaries directly; the .S files don't. +$(addprefix $(obj)/, $(firmware)): $(obj)/%.gen.o: $(fwdir)/% + +targets := $(patsubst $(obj)/%,%, \ + $(shell find $(obj) -name \*.gen.S 2>/dev/null)) diff --git a/drivers/base/firmware_loader/builtin/main.c b/drivers/base/firmware_loader/builtin/main.c new file mode 100644 index 000000000000..d36befebb1b9 --- /dev/null +++ b/drivers/base/firmware_loader/builtin/main.c @@ -0,0 +1,106 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Builtin firmware support */ + +#include <linux/firmware.h> +#include "../firmware.h" + +/* Only if FW_LOADER=y */ +#ifdef CONFIG_FW_LOADER + +struct builtin_fw { + char *name; + void *data; + unsigned long size; +}; + +extern struct builtin_fw __start_builtin_fw[]; +extern struct builtin_fw __end_builtin_fw[]; + +static bool fw_copy_to_prealloc_buf(struct firmware *fw, + void *buf, size_t size) +{ + if (!buf) + return true; + if (size < fw->size) + return false; + memcpy(buf, fw->data, fw->size); + return true; +} + +/** + * firmware_request_builtin() - load builtin firmware + * @fw: pointer to firmware struct + * @name: name of firmware file + * + * Some use cases in the kernel have a requirement so that no memory allocator + * is involved as these calls take place early in boot process. An example is + * the x86 CPU microcode loader. In these cases all the caller wants is to see + * if the firmware was built-in and if so use it right away. This can be used + * for such cases. + * + * This looks for the firmware in the built-in kernel. Only if the kernel was + * built-in with the firmware you are looking for will this return successfully. + * + * Callers of this API do not need to use release_firmware() as the pointer to + * the firmware is expected to be provided locally on the stack of the caller. + **/ +bool firmware_request_builtin(struct firmware *fw, const char *name) +{ + struct builtin_fw *b_fw; + + if (!fw) + return false; + + for (b_fw = __start_builtin_fw; b_fw != __end_builtin_fw; b_fw++) { + if (strcmp(name, b_fw->name) == 0) { + fw->size = b_fw->size; + fw->data = b_fw->data; + return true; + } + } + + return false; +} +EXPORT_SYMBOL_NS_GPL(firmware_request_builtin, "TEST_FIRMWARE"); + +/** + * firmware_request_builtin_buf() - load builtin firmware into optional buffer + * @fw: pointer to firmware struct + * @name: name of firmware file + * @buf: If set this lets you use a pre-allocated buffer so that the built-in + * firmware into is copied into. This field can be NULL. It is used by + * callers such as request_firmware_into_buf() and + * request_partial_firmware_into_buf() + * @size: if buf was provided, the max size of the allocated buffer available. + * If the built-in firmware does not fit into the pre-allocated @buf this + * call will fail. + * + * This looks for the firmware in the built-in kernel. Only if the kernel was + * built-in with the firmware you are looking for will this call possibly + * succeed. If you passed a @buf the firmware will be copied into it *iff* the + * built-in firmware fits into the pre-allocated buffer size specified in + * @size. + * + * This caller is to be used internally by the firmware_loader only. + **/ +bool firmware_request_builtin_buf(struct firmware *fw, const char *name, + void *buf, size_t size) +{ + if (!firmware_request_builtin(fw, name)) + return false; + + return fw_copy_to_prealloc_buf(fw, buf, size); +} + +bool firmware_is_builtin(const struct firmware *fw) +{ + struct builtin_fw *b_fw; + + for (b_fw = __start_builtin_fw; b_fw != __end_builtin_fw; b_fw++) + if (fw->data == b_fw->data) + return true; + + return false; +} + +#endif diff --git a/drivers/base/firmware_loader/fallback.c b/drivers/base/firmware_loader/fallback.c new file mode 100644 index 000000000000..3ef0b312ae71 --- /dev/null +++ b/drivers/base/firmware_loader/fallback.c @@ -0,0 +1,239 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/types.h> +#include <linux/kconfig.h> +#include <linux/list.h> +#include <linux/security.h> +#include <linux/umh.h> +#include <linux/sysctl.h> +#include <linux/module.h> + +#include "fallback.h" +#include "firmware.h" + +/* + * firmware fallback mechanism + */ + +/* + * use small loading timeout for caching devices' firmware because all these + * firmware images have been loaded successfully at lease once, also system is + * ready for completing firmware loading now. The maximum size of firmware in + * current distributions is about 2M bytes, so 10 secs should be enough. + */ +void fw_fallback_set_cache_timeout(void) +{ + fw_fallback_config.old_timeout = __firmware_loading_timeout(); + __fw_fallback_set_timeout(10); +} + +/* Restores the timeout to the value last configured during normal operation */ +void fw_fallback_set_default_timeout(void) +{ + __fw_fallback_set_timeout(fw_fallback_config.old_timeout); +} + +static long firmware_loading_timeout(void) +{ + return __firmware_loading_timeout() > 0 ? + __firmware_loading_timeout() * HZ : MAX_JIFFY_OFFSET; +} + +static inline int fw_sysfs_wait_timeout(struct fw_priv *fw_priv, long timeout) +{ + return __fw_state_wait_common(fw_priv, timeout); +} + +static LIST_HEAD(pending_fw_head); + +void kill_pending_fw_fallback_reqs(bool kill_all) +{ + struct fw_priv *fw_priv; + struct fw_priv *next; + + mutex_lock(&fw_lock); + list_for_each_entry_safe(fw_priv, next, &pending_fw_head, + pending_list) { + if (kill_all || !fw_priv->need_uevent) + __fw_load_abort(fw_priv); + } + + if (kill_all) + fw_load_abort_all = true; + + mutex_unlock(&fw_lock); +} + +/** + * fw_load_sysfs_fallback() - load a firmware via the sysfs fallback mechanism + * @fw_sysfs: firmware sysfs information for the firmware to load + * @timeout: timeout to wait for the load + * + * In charge of constructing a sysfs fallback interface for firmware loading. + **/ +static int fw_load_sysfs_fallback(struct fw_sysfs *fw_sysfs, long timeout) +{ + int retval = 0; + struct device *f_dev = &fw_sysfs->dev; + struct fw_priv *fw_priv = fw_sysfs->fw_priv; + + /* fall back on userspace loading */ + if (!fw_priv->data) + fw_priv->is_paged_buf = true; + + dev_set_uevent_suppress(f_dev, true); + + retval = device_add(f_dev); + if (retval) { + dev_err(f_dev, "%s: device_register failed\n", __func__); + goto err_put_dev; + } + + mutex_lock(&fw_lock); + if (fw_load_abort_all || fw_state_is_aborted(fw_priv)) { + mutex_unlock(&fw_lock); + retval = -EINTR; + goto out; + } + list_add(&fw_priv->pending_list, &pending_fw_head); + mutex_unlock(&fw_lock); + + if (fw_priv->opt_flags & FW_OPT_UEVENT) { + fw_priv->need_uevent = true; + dev_set_uevent_suppress(f_dev, false); + dev_dbg(f_dev, "firmware: requesting %s\n", fw_priv->fw_name); + kobject_uevent(&fw_sysfs->dev.kobj, KOBJ_ADD); + } else { + timeout = MAX_JIFFY_OFFSET; + } + + retval = fw_sysfs_wait_timeout(fw_priv, timeout); + if (retval < 0 && retval != -ENOENT) { + mutex_lock(&fw_lock); + fw_load_abort(fw_sysfs); + mutex_unlock(&fw_lock); + } + + if (fw_state_is_aborted(fw_priv)) { + if (retval == -ERESTARTSYS) + retval = -EINTR; + } else if (fw_priv->is_paged_buf && !fw_priv->data) + retval = -ENOMEM; + +out: + device_del(f_dev); +err_put_dev: + put_device(f_dev); + return retval; +} + +static int fw_load_from_user_helper(struct firmware *firmware, + const char *name, struct device *device, + u32 opt_flags) +{ + struct fw_sysfs *fw_sysfs; + long timeout; + int ret; + + timeout = firmware_loading_timeout(); + if (opt_flags & FW_OPT_NOWAIT) { + timeout = usermodehelper_read_lock_wait(timeout); + if (!timeout) { + dev_dbg(device, "firmware: %s loading timed out\n", + name); + return -EBUSY; + } + } else { + ret = usermodehelper_read_trylock(); + if (WARN_ON(ret)) { + dev_err(device, "firmware: %s will not be loaded\n", + name); + return ret; + } + } + + fw_sysfs = fw_create_instance(firmware, name, device, opt_flags); + if (IS_ERR(fw_sysfs)) { + ret = PTR_ERR(fw_sysfs); + goto out_unlock; + } + + fw_sysfs->fw_priv = firmware->priv; + ret = fw_load_sysfs_fallback(fw_sysfs, timeout); + + if (!ret) + ret = assign_fw(firmware, device); + +out_unlock: + usermodehelper_read_unlock(); + + return ret; +} + +static bool fw_force_sysfs_fallback(u32 opt_flags) +{ + if (fw_fallback_config.force_sysfs_fallback) + return true; + if (!(opt_flags & FW_OPT_USERHELPER)) + return false; + return true; +} + +static bool fw_run_sysfs_fallback(u32 opt_flags) +{ + int ret; + + if (fw_fallback_config.ignore_sysfs_fallback) { + pr_info_once("Ignoring firmware sysfs fallback due to sysctl knob\n"); + return false; + } + + if ((opt_flags & FW_OPT_NOFALLBACK_SYSFS)) + return false; + + /* Also permit LSMs and IMA to fail firmware sysfs fallback */ + ret = security_kernel_load_data(LOADING_FIRMWARE, true); + if (ret < 0) + return false; + + return fw_force_sysfs_fallback(opt_flags); +} + +/** + * firmware_fallback_sysfs() - use the fallback mechanism to find firmware + * @fw: pointer to firmware image + * @name: name of firmware file to look for + * @device: device for which firmware is being loaded + * @opt_flags: options to control firmware loading behaviour, as defined by + * &enum fw_opt + * @ret: return value from direct lookup which triggered the fallback mechanism + * + * This function is called if direct lookup for the firmware failed, it enables + * a fallback mechanism through userspace by exposing a sysfs loading + * interface. Userspace is in charge of loading the firmware through the sysfs + * loading interface. This sysfs fallback mechanism may be disabled completely + * on a system by setting the proc sysctl value ignore_sysfs_fallback to true. + * If this is false we check if the internal API caller set the + * @FW_OPT_NOFALLBACK_SYSFS flag, if so it would also disable the fallback + * mechanism. A system may want to enforce the sysfs fallback mechanism at all + * times, it can do this by setting ignore_sysfs_fallback to false and + * force_sysfs_fallback to true. + * Enabling force_sysfs_fallback is functionally equivalent to build a kernel + * with CONFIG_FW_LOADER_USER_HELPER_FALLBACK. + **/ +int firmware_fallback_sysfs(struct firmware *fw, const char *name, + struct device *device, + u32 opt_flags, + int ret) +{ + if (!fw_run_sysfs_fallback(opt_flags)) + return ret; + + if (!(opt_flags & FW_OPT_NO_WARN)) + dev_warn(device, "Falling back to sysfs fallback for: %s\n", + name); + else + dev_dbg(device, "Falling back to sysfs fallback for: %s\n", + name); + return fw_load_from_user_helper(fw, name, device, opt_flags); +} diff --git a/drivers/base/firmware_loader/fallback.h b/drivers/base/firmware_loader/fallback.h new file mode 100644 index 000000000000..ccf912bef6ca --- /dev/null +++ b/drivers/base/firmware_loader/fallback.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __FIRMWARE_FALLBACK_H +#define __FIRMWARE_FALLBACK_H + +#include <linux/firmware.h> +#include <linux/device.h> + +#include "firmware.h" +#include "sysfs.h" + +#ifdef CONFIG_FW_LOADER_USER_HELPER +int firmware_fallback_sysfs(struct firmware *fw, const char *name, + struct device *device, + u32 opt_flags, + int ret); +void kill_pending_fw_fallback_reqs(bool kill_all); + +void fw_fallback_set_cache_timeout(void); +void fw_fallback_set_default_timeout(void); + +#else /* CONFIG_FW_LOADER_USER_HELPER */ +static inline int firmware_fallback_sysfs(struct firmware *fw, const char *name, + struct device *device, + u32 opt_flags, + int ret) +{ + /* Keep carrying over the same error */ + return ret; +} + +static inline void kill_pending_fw_fallback_reqs(bool kill_all) { } +static inline void fw_fallback_set_cache_timeout(void) { } +static inline void fw_fallback_set_default_timeout(void) { } +#endif /* CONFIG_FW_LOADER_USER_HELPER */ + +#ifdef CONFIG_EFI_EMBEDDED_FIRMWARE +int firmware_fallback_platform(struct fw_priv *fw_priv); +#else +static inline int firmware_fallback_platform(struct fw_priv *fw_priv) +{ + return -ENOENT; +} +#endif + +#endif /* __FIRMWARE_FALLBACK_H */ diff --git a/drivers/base/firmware_loader/fallback_platform.c b/drivers/base/firmware_loader/fallback_platform.c new file mode 100644 index 000000000000..00af99f0aff2 --- /dev/null +++ b/drivers/base/firmware_loader/fallback_platform.c @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/efi_embedded_fw.h> +#include <linux/property.h> +#include <linux/security.h> +#include <linux/vmalloc.h> + +#include "fallback.h" +#include "firmware.h" + +int firmware_fallback_platform(struct fw_priv *fw_priv) +{ + const u8 *data; + size_t size; + int rc; + + if (!(fw_priv->opt_flags & FW_OPT_FALLBACK_PLATFORM)) + return -ENOENT; + + rc = security_kernel_load_data(LOADING_FIRMWARE, true); + if (rc) + return rc; + + rc = efi_get_embedded_fw(fw_priv->fw_name, &data, &size); + if (rc) + return rc; /* rc == -ENOENT when the fw was not found */ + + if (fw_priv->data && size > fw_priv->allocated_size) + return -ENOMEM; + + rc = security_kernel_post_load_data((u8 *)data, size, LOADING_FIRMWARE, + "platform"); + if (rc) + return rc; + + if (!fw_priv->data) + fw_priv->data = vmalloc(size); + if (!fw_priv->data) + return -ENOMEM; + + memcpy(fw_priv->data, data, size); + fw_priv->size = size; + fw_state_done(fw_priv); + return 0; +} diff --git a/drivers/base/firmware_loader/fallback_table.c b/drivers/base/firmware_loader/fallback_table.c new file mode 100644 index 000000000000..c8afc501a8a4 --- /dev/null +++ b/drivers/base/firmware_loader/fallback_table.c @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/types.h> +#include <linux/kconfig.h> +#include <linux/list.h> +#include <linux/slab.h> +#include <linux/export.h> +#include <linux/security.h> +#include <linux/highmem.h> +#include <linux/umh.h> +#include <linux/sysctl.h> + +#include "fallback.h" +#include "firmware.h" + +/* + * firmware fallback configuration table + */ + +struct firmware_fallback_config fw_fallback_config = { + .force_sysfs_fallback = IS_ENABLED(CONFIG_FW_LOADER_USER_HELPER_FALLBACK), + .loading_timeout = 60, + .old_timeout = 60, +}; +EXPORT_SYMBOL_NS_GPL(fw_fallback_config, "FIRMWARE_LOADER_PRIVATE"); + +#ifdef CONFIG_SYSCTL +static const struct ctl_table firmware_config_table[] = { + { + .procname = "force_sysfs_fallback", + .data = &fw_fallback_config.force_sysfs_fallback, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_douintvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, + { + .procname = "ignore_sysfs_fallback", + .data = &fw_fallback_config.ignore_sysfs_fallback, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_douintvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, +}; + +static struct ctl_table_header *firmware_config_sysct_table_header; +int register_firmware_config_sysctl(void) +{ + firmware_config_sysct_table_header = + register_sysctl("kernel/firmware_config", + firmware_config_table); + if (!firmware_config_sysct_table_header) + return -ENOMEM; + return 0; +} +EXPORT_SYMBOL_NS_GPL(register_firmware_config_sysctl, "FIRMWARE_LOADER_PRIVATE"); + +void unregister_firmware_config_sysctl(void) +{ + unregister_sysctl_table(firmware_config_sysct_table_header); + firmware_config_sysct_table_header = NULL; +} +EXPORT_SYMBOL_NS_GPL(unregister_firmware_config_sysctl, "FIRMWARE_LOADER_PRIVATE"); + +#endif /* CONFIG_SYSCTL */ diff --git a/drivers/base/firmware_loader/firmware.h b/drivers/base/firmware_loader/firmware.h new file mode 100644 index 000000000000..e891742ba264 --- /dev/null +++ b/drivers/base/firmware_loader/firmware.h @@ -0,0 +1,198 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __FIRMWARE_LOADER_H +#define __FIRMWARE_LOADER_H + +#include <linux/bitops.h> +#include <linux/firmware.h> +#include <linux/types.h> +#include <linux/kref.h> +#include <linux/list.h> +#include <linux/completion.h> + +/** + * enum fw_opt - options to control firmware loading behaviour + * + * @FW_OPT_UEVENT: Enables the fallback mechanism to send a kobject uevent + * when the firmware is not found. Userspace is in charge to load the + * firmware using the sysfs loading facility. + * @FW_OPT_NOWAIT: Used to describe the firmware request is asynchronous. + * @FW_OPT_USERHELPER: Enable the fallback mechanism, in case the direct + * filesystem lookup fails at finding the firmware. For details refer to + * firmware_fallback_sysfs(). + * @FW_OPT_NO_WARN: Quiet, avoid printing warning messages. + * @FW_OPT_NOCACHE: Disables firmware caching. Firmware caching is used to + * cache the firmware upon suspend, so that upon resume races against the + * firmware file lookup on storage is avoided. Used for calls where the + * file may be too big, or where the driver takes charge of its own + * firmware caching mechanism. + * @FW_OPT_NOFALLBACK_SYSFS: Disable the sysfs fallback mechanism. Takes + * precedence over &FW_OPT_UEVENT and &FW_OPT_USERHELPER. + * @FW_OPT_FALLBACK_PLATFORM: Enable fallback to device fw copy embedded in + * the platform's main firmware. If both this fallback and the sysfs + * fallback are enabled, then this fallback will be tried first. + * @FW_OPT_PARTIAL: Allow partial read of firmware instead of needing to read + * entire file. + */ +enum fw_opt { + FW_OPT_UEVENT = BIT(0), + FW_OPT_NOWAIT = BIT(1), + FW_OPT_USERHELPER = BIT(2), + FW_OPT_NO_WARN = BIT(3), + FW_OPT_NOCACHE = BIT(4), + FW_OPT_NOFALLBACK_SYSFS = BIT(5), + FW_OPT_FALLBACK_PLATFORM = BIT(6), + FW_OPT_PARTIAL = BIT(7), +}; + +enum fw_status { + FW_STATUS_UNKNOWN, + FW_STATUS_LOADING, + FW_STATUS_DONE, + FW_STATUS_ABORTED, +}; + +/* + * Concurrent request_firmware() for the same firmware need to be + * serialized. struct fw_state is simple state machine which hold the + * state of the firmware loading. + */ +struct fw_state { + struct completion completion; + enum fw_status status; +}; + +struct fw_priv { + struct kref ref; + struct list_head list; + struct firmware_cache *fwc; + struct fw_state fw_st; + void *data; + size_t size; + size_t allocated_size; + size_t offset; + u32 opt_flags; +#ifdef CONFIG_FW_LOADER_PAGED_BUF + bool is_paged_buf; + struct page **pages; + int nr_pages; + int page_array_size; +#endif +#ifdef CONFIG_FW_LOADER_USER_HELPER + bool need_uevent; + struct list_head pending_list; +#endif + const char *fw_name; +}; + +extern struct mutex fw_lock; +extern struct firmware_cache fw_cache; +extern bool fw_load_abort_all; + +static inline bool __fw_state_check(struct fw_priv *fw_priv, + enum fw_status status) +{ + struct fw_state *fw_st = &fw_priv->fw_st; + + return fw_st->status == status; +} + +static inline int __fw_state_wait_common(struct fw_priv *fw_priv, long timeout) +{ + struct fw_state *fw_st = &fw_priv->fw_st; + long ret; + + ret = wait_for_completion_killable_timeout(&fw_st->completion, timeout); + if (ret != 0 && fw_st->status == FW_STATUS_ABORTED) + return -ENOENT; + if (!ret) + return -ETIMEDOUT; + + return ret < 0 ? ret : 0; +} + +static inline void __fw_state_set(struct fw_priv *fw_priv, + enum fw_status status) +{ + struct fw_state *fw_st = &fw_priv->fw_st; + + WRITE_ONCE(fw_st->status, status); + + if (status == FW_STATUS_DONE || status == FW_STATUS_ABORTED) { +#ifdef CONFIG_FW_LOADER_USER_HELPER + /* + * Doing this here ensures that the fw_priv is deleted from + * the pending list in all abort/done paths. + */ + list_del_init(&fw_priv->pending_list); +#endif + complete_all(&fw_st->completion); + } +} + +static inline void fw_state_aborted(struct fw_priv *fw_priv) +{ + __fw_state_set(fw_priv, FW_STATUS_ABORTED); +} + +static inline bool fw_state_is_aborted(struct fw_priv *fw_priv) +{ + return __fw_state_check(fw_priv, FW_STATUS_ABORTED); +} + +static inline void fw_state_start(struct fw_priv *fw_priv) +{ + __fw_state_set(fw_priv, FW_STATUS_LOADING); +} + +static inline void fw_state_done(struct fw_priv *fw_priv) +{ + __fw_state_set(fw_priv, FW_STATUS_DONE); +} + +static inline bool fw_state_is_done(struct fw_priv *fw_priv) +{ + return __fw_state_check(fw_priv, FW_STATUS_DONE); +} + +static inline bool fw_state_is_loading(struct fw_priv *fw_priv) +{ + return __fw_state_check(fw_priv, FW_STATUS_LOADING); +} + +int alloc_lookup_fw_priv(const char *fw_name, struct firmware_cache *fwc, + struct fw_priv **fw_priv, void *dbuf, size_t size, + size_t offset, u32 opt_flags); +int assign_fw(struct firmware *fw, struct device *device); +void free_fw_priv(struct fw_priv *fw_priv); +void fw_state_init(struct fw_priv *fw_priv); + +#ifdef CONFIG_FW_LOADER +bool firmware_is_builtin(const struct firmware *fw); +bool firmware_request_builtin_buf(struct firmware *fw, const char *name, + void *buf, size_t size); +#else /* module case */ +static inline bool firmware_is_builtin(const struct firmware *fw) +{ + return false; +} +static inline bool firmware_request_builtin_buf(struct firmware *fw, + const char *name, + void *buf, size_t size) +{ + return false; +} +#endif + +#ifdef CONFIG_FW_LOADER_PAGED_BUF +void fw_free_paged_buf(struct fw_priv *fw_priv); +int fw_grow_paged_buf(struct fw_priv *fw_priv, int pages_needed); +int fw_map_paged_buf(struct fw_priv *fw_priv); +bool fw_is_paged_buf(struct fw_priv *fw_priv); +#else +static inline void fw_free_paged_buf(struct fw_priv *fw_priv) {} +static inline int fw_grow_paged_buf(struct fw_priv *fw_priv, int pages_needed) { return -ENXIO; } +static inline int fw_map_paged_buf(struct fw_priv *fw_priv) { return -ENXIO; } +static inline bool fw_is_paged_buf(struct fw_priv *fw_priv) { return false; } +#endif + +#endif /* __FIRMWARE_LOADER_H */ diff --git a/drivers/base/firmware_loader/main.c b/drivers/base/firmware_loader/main.c new file mode 100644 index 000000000000..4ebdca9e4da4 --- /dev/null +++ b/drivers/base/firmware_loader/main.c @@ -0,0 +1,1684 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * main.c - Multi purpose firmware loading support + * + * Copyright (c) 2003 Manuel Estrada Sainz + * + * Please see Documentation/driver-api/firmware/ for more information. + * + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/capability.h> +#include <linux/device.h> +#include <linux/kernel_read_file.h> +#include <linux/module.h> +#include <linux/init.h> +#include <linux/initrd.h> +#include <linux/timer.h> +#include <linux/vmalloc.h> +#include <linux/interrupt.h> +#include <linux/bitops.h> +#include <linux/mutex.h> +#include <linux/workqueue.h> +#include <linux/highmem.h> +#include <linux/firmware.h> +#include <linux/slab.h> +#include <linux/sched.h> +#include <linux/file.h> +#include <linux/list.h> +#include <linux/fs.h> +#include <linux/async.h> +#include <linux/pm.h> +#include <linux/suspend.h> +#include <linux/syscore_ops.h> +#include <linux/reboot.h> +#include <linux/security.h> +#include <linux/zstd.h> +#include <linux/xz.h> + +#include <generated/utsrelease.h> + +#include "../base.h" +#include "firmware.h" +#include "fallback.h" + +MODULE_AUTHOR("Manuel Estrada Sainz"); +MODULE_DESCRIPTION("Multi purpose firmware loading support"); +MODULE_LICENSE("GPL"); + +struct firmware_cache { + /* firmware_buf instance will be added into the below list */ + spinlock_t lock; + struct list_head head; + int state; + +#ifdef CONFIG_FW_CACHE + /* + * Names of firmware images which have been cached successfully + * will be added into the below list so that device uncache + * helper can trace which firmware images have been cached + * before. + */ + spinlock_t name_lock; + struct list_head fw_names; + + struct delayed_work work; + + struct notifier_block pm_notify; +#endif +}; + +struct fw_cache_entry { + struct list_head list; + const char *name; +}; + +struct fw_name_devm { + unsigned long magic; + const char *name; +}; + +static inline struct fw_priv *to_fw_priv(struct kref *ref) +{ + return container_of(ref, struct fw_priv, ref); +} + +#define FW_LOADER_NO_CACHE 0 +#define FW_LOADER_START_CACHE 1 + +/* fw_lock could be moved to 'struct fw_sysfs' but since it is just + * guarding for corner cases a global lock should be OK */ +DEFINE_MUTEX(fw_lock); + +struct firmware_cache fw_cache; +bool fw_load_abort_all; + +void fw_state_init(struct fw_priv *fw_priv) +{ + struct fw_state *fw_st = &fw_priv->fw_st; + + init_completion(&fw_st->completion); + fw_st->status = FW_STATUS_UNKNOWN; +} + +static inline int fw_state_wait(struct fw_priv *fw_priv) +{ + return __fw_state_wait_common(fw_priv, MAX_SCHEDULE_TIMEOUT); +} + +static void fw_cache_piggyback_on_request(struct fw_priv *fw_priv); + +static struct fw_priv *__allocate_fw_priv(const char *fw_name, + struct firmware_cache *fwc, + void *dbuf, + size_t size, + size_t offset, + u32 opt_flags) +{ + struct fw_priv *fw_priv; + + /* For a partial read, the buffer must be preallocated. */ + if ((opt_flags & FW_OPT_PARTIAL) && !dbuf) + return NULL; + + /* Only partial reads are allowed to use an offset. */ + if (offset != 0 && !(opt_flags & FW_OPT_PARTIAL)) + return NULL; + + fw_priv = kzalloc(sizeof(*fw_priv), GFP_ATOMIC); + if (!fw_priv) + return NULL; + + fw_priv->fw_name = kstrdup_const(fw_name, GFP_ATOMIC); + if (!fw_priv->fw_name) { + kfree(fw_priv); + return NULL; + } + + kref_init(&fw_priv->ref); + fw_priv->fwc = fwc; + fw_priv->data = dbuf; + fw_priv->allocated_size = size; + fw_priv->offset = offset; + fw_priv->opt_flags = opt_flags; + fw_state_init(fw_priv); +#ifdef CONFIG_FW_LOADER_USER_HELPER + INIT_LIST_HEAD(&fw_priv->pending_list); +#endif + + pr_debug("%s: fw-%s fw_priv=%p\n", __func__, fw_name, fw_priv); + + return fw_priv; +} + +static struct fw_priv *__lookup_fw_priv(const char *fw_name) +{ + struct fw_priv *tmp; + struct firmware_cache *fwc = &fw_cache; + + list_for_each_entry(tmp, &fwc->head, list) + if (!strcmp(tmp->fw_name, fw_name)) + return tmp; + return NULL; +} + +/* Returns 1 for batching firmware requests with the same name */ +int alloc_lookup_fw_priv(const char *fw_name, struct firmware_cache *fwc, + struct fw_priv **fw_priv, void *dbuf, size_t size, + size_t offset, u32 opt_flags) +{ + struct fw_priv *tmp; + + spin_lock(&fwc->lock); + /* + * Do not merge requests that are marked to be non-cached or + * are performing partial reads. + */ + if (!(opt_flags & (FW_OPT_NOCACHE | FW_OPT_PARTIAL))) { + tmp = __lookup_fw_priv(fw_name); + if (tmp) { + kref_get(&tmp->ref); + spin_unlock(&fwc->lock); + *fw_priv = tmp; + pr_debug("batched request - sharing the same struct fw_priv and lookup for multiple requests\n"); + return 1; + } + } + + tmp = __allocate_fw_priv(fw_name, fwc, dbuf, size, offset, opt_flags); + if (tmp) { + INIT_LIST_HEAD(&tmp->list); + if (!(opt_flags & FW_OPT_NOCACHE)) + list_add(&tmp->list, &fwc->head); + } + spin_unlock(&fwc->lock); + + *fw_priv = tmp; + + return tmp ? 0 : -ENOMEM; +} + +static void __free_fw_priv(struct kref *ref) + __releases(&fwc->lock) +{ + struct fw_priv *fw_priv = to_fw_priv(ref); + struct firmware_cache *fwc = fw_priv->fwc; + + pr_debug("%s: fw-%s fw_priv=%p data=%p size=%u\n", + __func__, fw_priv->fw_name, fw_priv, fw_priv->data, + (unsigned int)fw_priv->size); + + list_del(&fw_priv->list); + spin_unlock(&fwc->lock); + + if (fw_is_paged_buf(fw_priv)) + fw_free_paged_buf(fw_priv); + else if (!fw_priv->allocated_size) + vfree(fw_priv->data); + + kfree_const(fw_priv->fw_name); + kfree(fw_priv); +} + +void free_fw_priv(struct fw_priv *fw_priv) +{ + struct firmware_cache *fwc = fw_priv->fwc; + spin_lock(&fwc->lock); + if (!kref_put(&fw_priv->ref, __free_fw_priv)) + spin_unlock(&fwc->lock); +} + +#ifdef CONFIG_FW_LOADER_PAGED_BUF +bool fw_is_paged_buf(struct fw_priv *fw_priv) +{ + return fw_priv->is_paged_buf; +} + +void fw_free_paged_buf(struct fw_priv *fw_priv) +{ + int i; + + if (!fw_priv->pages) + return; + + vunmap(fw_priv->data); + + for (i = 0; i < fw_priv->nr_pages; i++) + __free_page(fw_priv->pages[i]); + kvfree(fw_priv->pages); + fw_priv->pages = NULL; + fw_priv->page_array_size = 0; + fw_priv->nr_pages = 0; + fw_priv->data = NULL; + fw_priv->size = 0; +} + +int fw_grow_paged_buf(struct fw_priv *fw_priv, int pages_needed) +{ + /* If the array of pages is too small, grow it */ + if (fw_priv->page_array_size < pages_needed) { + int new_array_size = max(pages_needed, + fw_priv->page_array_size * 2); + struct page **new_pages; + + new_pages = kvmalloc_array(new_array_size, sizeof(void *), + GFP_KERNEL); + if (!new_pages) + return -ENOMEM; + memcpy(new_pages, fw_priv->pages, + fw_priv->page_array_size * sizeof(void *)); + memset(&new_pages[fw_priv->page_array_size], 0, sizeof(void *) * + (new_array_size - fw_priv->page_array_size)); + kvfree(fw_priv->pages); + fw_priv->pages = new_pages; + fw_priv->page_array_size = new_array_size; + } + + while (fw_priv->nr_pages < pages_needed) { + fw_priv->pages[fw_priv->nr_pages] = + alloc_page(GFP_KERNEL | __GFP_HIGHMEM); + + if (!fw_priv->pages[fw_priv->nr_pages]) + return -ENOMEM; + fw_priv->nr_pages++; + } + + return 0; +} + +int fw_map_paged_buf(struct fw_priv *fw_priv) +{ + /* one pages buffer should be mapped/unmapped only once */ + if (!fw_priv->pages) + return 0; + + vunmap(fw_priv->data); + fw_priv->data = vmap(fw_priv->pages, fw_priv->nr_pages, 0, + PAGE_KERNEL_RO); + if (!fw_priv->data) + return -ENOMEM; + + return 0; +} +#endif + +/* + * ZSTD-compressed firmware support + */ +#ifdef CONFIG_FW_LOADER_COMPRESS_ZSTD +static int fw_decompress_zstd(struct device *dev, struct fw_priv *fw_priv, + size_t in_size, const void *in_buffer) +{ + size_t len, out_size, workspace_size; + void *workspace, *out_buf; + zstd_dctx *ctx; + int err; + + if (fw_priv->allocated_size) { + out_size = fw_priv->allocated_size; + out_buf = fw_priv->data; + } else { + zstd_frame_header params; + + if (zstd_get_frame_header(¶ms, in_buffer, in_size) || + params.frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN) { + dev_dbg(dev, "%s: invalid zstd header\n", __func__); + return -EINVAL; + } + out_size = params.frameContentSize; + out_buf = vzalloc(out_size); + if (!out_buf) + return -ENOMEM; + } + + workspace_size = zstd_dctx_workspace_bound(); + workspace = kvzalloc(workspace_size, GFP_KERNEL); + if (!workspace) { + err = -ENOMEM; + goto error; + } + + ctx = zstd_init_dctx(workspace, workspace_size); + if (!ctx) { + dev_dbg(dev, "%s: failed to initialize context\n", __func__); + err = -EINVAL; + goto error; + } + + len = zstd_decompress_dctx(ctx, out_buf, out_size, in_buffer, in_size); + if (zstd_is_error(len)) { + dev_dbg(dev, "%s: failed to decompress: %d\n", __func__, + zstd_get_error_code(len)); + err = -EINVAL; + goto error; + } + + if (!fw_priv->allocated_size) + fw_priv->data = out_buf; + fw_priv->size = len; + err = 0; + + error: + kvfree(workspace); + if (err && !fw_priv->allocated_size) + vfree(out_buf); + return err; +} +#endif /* CONFIG_FW_LOADER_COMPRESS_ZSTD */ + +/* + * XZ-compressed firmware support + */ +#ifdef CONFIG_FW_LOADER_COMPRESS_XZ +/* show an error and return the standard error code */ +static int fw_decompress_xz_error(struct device *dev, enum xz_ret xz_ret) +{ + if (xz_ret != XZ_STREAM_END) { + dev_warn(dev, "xz decompression failed (xz_ret=%d)\n", xz_ret); + return xz_ret == XZ_MEM_ERROR ? -ENOMEM : -EINVAL; + } + return 0; +} + +/* single-shot decompression onto the pre-allocated buffer */ +static int fw_decompress_xz_single(struct device *dev, struct fw_priv *fw_priv, + size_t in_size, const void *in_buffer) +{ + struct xz_dec *xz_dec; + struct xz_buf xz_buf; + enum xz_ret xz_ret; + + xz_dec = xz_dec_init(XZ_SINGLE, (u32)-1); + if (!xz_dec) + return -ENOMEM; + + xz_buf.in_size = in_size; + xz_buf.in = in_buffer; + xz_buf.in_pos = 0; + xz_buf.out_size = fw_priv->allocated_size; + xz_buf.out = fw_priv->data; + xz_buf.out_pos = 0; + + xz_ret = xz_dec_run(xz_dec, &xz_buf); + xz_dec_end(xz_dec); + + fw_priv->size = xz_buf.out_pos; + return fw_decompress_xz_error(dev, xz_ret); +} + +/* decompression on paged buffer and map it */ +static int fw_decompress_xz_pages(struct device *dev, struct fw_priv *fw_priv, + size_t in_size, const void *in_buffer) +{ + struct xz_dec *xz_dec; + struct xz_buf xz_buf; + enum xz_ret xz_ret; + struct page *page; + int err = 0; + + xz_dec = xz_dec_init(XZ_DYNALLOC, (u32)-1); + if (!xz_dec) + return -ENOMEM; + + xz_buf.in_size = in_size; + xz_buf.in = in_buffer; + xz_buf.in_pos = 0; + + fw_priv->is_paged_buf = true; + fw_priv->size = 0; + do { + if (fw_grow_paged_buf(fw_priv, fw_priv->nr_pages + 1)) { + err = -ENOMEM; + goto out; + } + + /* decompress onto the new allocated page */ + page = fw_priv->pages[fw_priv->nr_pages - 1]; + xz_buf.out = kmap_local_page(page); + xz_buf.out_pos = 0; + xz_buf.out_size = PAGE_SIZE; + xz_ret = xz_dec_run(xz_dec, &xz_buf); + kunmap_local(xz_buf.out); + fw_priv->size += xz_buf.out_pos; + /* partial decompression means either end or error */ + if (xz_buf.out_pos != PAGE_SIZE) + break; + } while (xz_ret == XZ_OK); + + err = fw_decompress_xz_error(dev, xz_ret); + if (!err) + err = fw_map_paged_buf(fw_priv); + + out: + xz_dec_end(xz_dec); + return err; +} + +static int fw_decompress_xz(struct device *dev, struct fw_priv *fw_priv, + size_t in_size, const void *in_buffer) +{ + /* if the buffer is pre-allocated, we can perform in single-shot mode */ + if (fw_priv->data) + return fw_decompress_xz_single(dev, fw_priv, in_size, in_buffer); + else + return fw_decompress_xz_pages(dev, fw_priv, in_size, in_buffer); +} +#endif /* CONFIG_FW_LOADER_COMPRESS_XZ */ + +/* direct firmware loading support */ +static char fw_path_para[256]; +static const char * const fw_path[] = { + fw_path_para, + "/lib/firmware/updates/" UTS_RELEASE, + "/lib/firmware/updates", + "/lib/firmware/" UTS_RELEASE, + "/lib/firmware" +}; + +/* + * Typical usage is that passing 'firmware_class.path=$CUSTOMIZED_PATH' + * from kernel command line because firmware_class is generally built in + * kernel instead of module. + */ +module_param_string(path, fw_path_para, sizeof(fw_path_para), 0644); +MODULE_PARM_DESC(path, "customized firmware image search path with a higher priority than default path"); + +static int +fw_get_filesystem_firmware(struct device *device, struct fw_priv *fw_priv, + const char *suffix, + int (*decompress)(struct device *dev, + struct fw_priv *fw_priv, + size_t in_size, + const void *in_buffer)) +{ + size_t size; + int i, len, maxlen = 0; + int rc = -ENOENT; + char *path, *nt = NULL; + size_t msize = INT_MAX; + void *buffer = NULL; + + /* Already populated data member means we're loading into a buffer */ + if (!decompress && fw_priv->data) { + buffer = fw_priv->data; + msize = fw_priv->allocated_size; + } + + path = __getname(); + if (!path) + return -ENOMEM; + + wait_for_initramfs(); + for (i = 0; i < ARRAY_SIZE(fw_path); i++) { + size_t file_size = 0; + size_t *file_size_ptr = NULL; + + /* skip the unset customized path */ + if (!fw_path[i][0]) + continue; + + /* strip off \n from customized path */ + maxlen = strlen(fw_path[i]); + if (i == 0) { + nt = strchr(fw_path[i], '\n'); + if (nt) + maxlen = nt - fw_path[i]; + } + + len = snprintf(path, PATH_MAX, "%.*s/%s%s", + maxlen, fw_path[i], + fw_priv->fw_name, suffix); + if (len >= PATH_MAX) { + rc = -ENAMETOOLONG; + break; + } + + fw_priv->size = 0; + + /* + * The total file size is only examined when doing a partial + * read; the "full read" case needs to fail if the whole + * firmware was not completely loaded. + */ + if ((fw_priv->opt_flags & FW_OPT_PARTIAL) && buffer) + file_size_ptr = &file_size; + + /* load firmware files from the mount namespace of init */ + rc = kernel_read_file_from_path_initns(path, fw_priv->offset, + &buffer, msize, + file_size_ptr, + READING_FIRMWARE); + if (rc < 0) { + if (!(fw_priv->opt_flags & FW_OPT_NO_WARN)) { + if (rc != -ENOENT) + dev_warn(device, + "loading %s failed with error %d\n", + path, rc); + else + dev_dbg(device, + "loading %s failed for no such file or directory.\n", + path); + } + continue; + } + size = rc; + rc = 0; + + dev_dbg(device, "Loading firmware from %s\n", path); + if (decompress) { + dev_dbg(device, "f/w decompressing %s\n", + fw_priv->fw_name); + rc = decompress(device, fw_priv, size, buffer); + /* discard the superfluous original content */ + vfree(buffer); + buffer = NULL; + if (rc) { + fw_free_paged_buf(fw_priv); + continue; + } + } else { + dev_dbg(device, "direct-loading %s\n", + fw_priv->fw_name); + if (!fw_priv->data) + fw_priv->data = buffer; + fw_priv->size = size; + } + fw_state_done(fw_priv); + break; + } + __putname(path); + + return rc; +} + +/* firmware holds the ownership of pages */ +static void firmware_free_data(const struct firmware *fw) +{ + /* Loaded directly? */ + if (!fw->priv) { + vfree(fw->data); + return; + } + free_fw_priv(fw->priv); +} + +/* store the pages buffer info firmware from buf */ +static void fw_set_page_data(struct fw_priv *fw_priv, struct firmware *fw) +{ + fw->priv = fw_priv; + fw->size = fw_priv->size; + fw->data = fw_priv->data; + + pr_debug("%s: fw-%s fw_priv=%p data=%p size=%u\n", + __func__, fw_priv->fw_name, fw_priv, fw_priv->data, + (unsigned int)fw_priv->size); +} + +#ifdef CONFIG_FW_CACHE +static void fw_name_devm_release(struct device *dev, void *res) +{ + struct fw_name_devm *fwn = res; + + if (fwn->magic == (unsigned long)&fw_cache) + pr_debug("%s: fw_name-%s devm-%p released\n", + __func__, fwn->name, res); + kfree_const(fwn->name); +} + +static int fw_devm_match(struct device *dev, void *res, + void *match_data) +{ + struct fw_name_devm *fwn = res; + + return (fwn->magic == (unsigned long)&fw_cache) && + !strcmp(fwn->name, match_data); +} + +static struct fw_name_devm *fw_find_devm_name(struct device *dev, + const char *name) +{ + struct fw_name_devm *fwn; + + fwn = devres_find(dev, fw_name_devm_release, + fw_devm_match, (void *)name); + return fwn; +} + +static bool fw_cache_is_setup(struct device *dev, const char *name) +{ + struct fw_name_devm *fwn; + + fwn = fw_find_devm_name(dev, name); + if (fwn) + return true; + + return false; +} + +/* add firmware name into devres list */ +static int fw_add_devm_name(struct device *dev, const char *name) +{ + struct fw_name_devm *fwn; + + if (fw_cache_is_setup(dev, name)) + return 0; + + fwn = devres_alloc(fw_name_devm_release, sizeof(struct fw_name_devm), + GFP_KERNEL); + if (!fwn) + return -ENOMEM; + fwn->name = kstrdup_const(name, GFP_KERNEL); + if (!fwn->name) { + devres_free(fwn); + return -ENOMEM; + } + + fwn->magic = (unsigned long)&fw_cache; + devres_add(dev, fwn); + + return 0; +} +#else +static bool fw_cache_is_setup(struct device *dev, const char *name) +{ + return false; +} + +static int fw_add_devm_name(struct device *dev, const char *name) +{ + return 0; +} +#endif + +int assign_fw(struct firmware *fw, struct device *device) +{ + struct fw_priv *fw_priv = fw->priv; + int ret; + + mutex_lock(&fw_lock); + if (!fw_priv->size || fw_state_is_aborted(fw_priv)) { + mutex_unlock(&fw_lock); + return -ENOENT; + } + + /* + * add firmware name into devres list so that we can auto cache + * and uncache firmware for device. + * + * device may has been deleted already, but the problem + * should be fixed in devres or driver core. + */ + /* don't cache firmware handled without uevent */ + if (device && (fw_priv->opt_flags & FW_OPT_UEVENT) && + !(fw_priv->opt_flags & FW_OPT_NOCACHE)) { + ret = fw_add_devm_name(device, fw_priv->fw_name); + if (ret) { + mutex_unlock(&fw_lock); + return ret; + } + } + + /* + * After caching firmware image is started, let it piggyback + * on request firmware. + */ + if (!(fw_priv->opt_flags & FW_OPT_NOCACHE) && + fw_priv->fwc->state == FW_LOADER_START_CACHE) + fw_cache_piggyback_on_request(fw_priv); + + /* pass the pages buffer to driver at the last minute */ + fw_set_page_data(fw_priv, fw); + mutex_unlock(&fw_lock); + return 0; +} + +/* prepare firmware and firmware_buf structs; + * return 0 if a firmware is already assigned, 1 if need to load one, + * or a negative error code + */ +static int +_request_firmware_prepare(struct firmware **firmware_p, const char *name, + struct device *device, void *dbuf, size_t size, + size_t offset, u32 opt_flags) +{ + struct firmware *firmware; + struct fw_priv *fw_priv; + int ret; + + *firmware_p = firmware = kzalloc(sizeof(*firmware), GFP_KERNEL); + if (!firmware) { + dev_err(device, "%s: kmalloc(struct firmware) failed\n", + __func__); + return -ENOMEM; + } + + if (firmware_request_builtin_buf(firmware, name, dbuf, size)) { + dev_dbg(device, "using built-in %s\n", name); + return 0; /* assigned */ + } + + ret = alloc_lookup_fw_priv(name, &fw_cache, &fw_priv, dbuf, size, + offset, opt_flags); + + /* + * bind with 'priv' now to avoid warning in failure path + * of requesting firmware. + */ + firmware->priv = fw_priv; + + if (ret > 0) { + ret = fw_state_wait(fw_priv); + if (!ret) { + fw_set_page_data(fw_priv, firmware); + return 0; /* assigned */ + } + } + + if (ret < 0) + return ret; + return 1; /* need to load */ +} + +/* + * Batched requests need only one wake, we need to do this step last due to the + * fallback mechanism. The buf is protected with kref_get(), and it won't be + * released until the last user calls release_firmware(). + * + * Failed batched requests are possible as well, in such cases we just share + * the struct fw_priv and won't release it until all requests are woken + * and have gone through this same path. + */ +static void fw_abort_batch_reqs(struct firmware *fw) +{ + struct fw_priv *fw_priv; + + /* Loaded directly? */ + if (!fw || !fw->priv) + return; + + fw_priv = fw->priv; + mutex_lock(&fw_lock); + if (!fw_state_is_aborted(fw_priv)) + fw_state_aborted(fw_priv); + mutex_unlock(&fw_lock); +} + +#if defined(CONFIG_FW_LOADER_DEBUG) +#include <crypto/sha2.h> + +static void fw_log_firmware_info(const struct firmware *fw, const char *name, struct device *device) +{ + u8 digest[SHA256_DIGEST_SIZE]; + + sha256(fw->data, fw->size, digest); + dev_dbg(device, "Loaded FW: %s, sha256: %*phN\n", + name, SHA256_DIGEST_SIZE, digest); +} +#else +static void fw_log_firmware_info(const struct firmware *fw, const char *name, + struct device *device) +{} +#endif + +/* called from request_firmware() and request_firmware_work_func() */ +static int +_request_firmware(const struct firmware **firmware_p, const char *name, + struct device *device, void *buf, size_t size, + size_t offset, u32 opt_flags) +{ + struct firmware *fw = NULL; + bool nondirect = false; + int ret; + + if (!firmware_p) + return -EINVAL; + + if (!name || name[0] == '\0') { + ret = -EINVAL; + goto out; + } + + + /* + * Reject firmware file names with ".." path components. + * There are drivers that construct firmware file names from + * device-supplied strings, and we don't want some device to be + * able to tell us "I would like to be sent my firmware from + * ../../../etc/shadow, please". + * + * This intentionally only looks at the firmware name, not at + * the firmware base directory or at symlink contents. + */ + if (name_contains_dotdot(name)) { + dev_warn(device, + "Firmware load for '%s' refused, path contains '..' component\n", + name); + ret = -EINVAL; + goto out; + } + + ret = _request_firmware_prepare(&fw, name, device, buf, size, + offset, opt_flags); + if (ret <= 0) /* error or already assigned */ + goto out; + + /* + * We are about to try to access the firmware file. Because we may have been + * called by a driver when serving an unrelated request from userland, we use + * the kernel credentials to read the file. + */ + scoped_with_kernel_creds() { + ret = fw_get_filesystem_firmware(device, fw->priv, "", NULL); + + /* Only full reads can support decompression, platform, and sysfs. */ + if (!(opt_flags & FW_OPT_PARTIAL)) + nondirect = true; + +#ifdef CONFIG_FW_LOADER_COMPRESS_ZSTD + if (ret == -ENOENT && nondirect) + ret = fw_get_filesystem_firmware(device, fw->priv, ".zst", + fw_decompress_zstd); +#endif +#ifdef CONFIG_FW_LOADER_COMPRESS_XZ + if (ret == -ENOENT && nondirect) + ret = fw_get_filesystem_firmware(device, fw->priv, ".xz", + fw_decompress_xz); +#endif + if (ret == -ENOENT && nondirect) + ret = firmware_fallback_platform(fw->priv); + + if (ret) { + if (!(opt_flags & FW_OPT_NO_WARN)) + dev_warn(device, + "Direct firmware load for %s failed with error %d\n", + name, ret); + if (nondirect) + ret = firmware_fallback_sysfs(fw, name, device, + opt_flags, ret); + } else { + ret = assign_fw(fw, device); + } + } + +out: + if (ret < 0) { + fw_abort_batch_reqs(fw); + release_firmware(fw); + fw = NULL; + } else { + fw_log_firmware_info(fw, name, device); + } + + *firmware_p = fw; + return ret; +} + +/** + * request_firmware() - send firmware request and wait for it + * @firmware_p: pointer to firmware image + * @name: name of firmware file + * @device: device for which firmware is being loaded + * + * @firmware_p will be used to return a firmware image by the name + * of @name for device @device. + * + * Should be called from user context where sleeping is allowed. + * + * @name will be used as $FIRMWARE in the uevent environment and + * should be distinctive enough not to be confused with any other + * firmware image for this or any other device. + * It must not contain any ".." path components - "foo/bar..bin" is + * allowed, but "foo/../bar.bin" is not. + * + * Caller must hold the reference count of @device. + * + * The function can be called safely inside device's suspend and + * resume callback. + **/ +int +request_firmware(const struct firmware **firmware_p, const char *name, + struct device *device) +{ + int ret; + + /* Need to pin this module until return */ + __module_get(THIS_MODULE); + ret = _request_firmware(firmware_p, name, device, NULL, 0, 0, + FW_OPT_UEVENT); + module_put(THIS_MODULE); + return ret; +} +EXPORT_SYMBOL(request_firmware); + +/** + * firmware_request_nowarn() - request for an optional fw module + * @firmware: pointer to firmware image + * @name: name of firmware file + * @device: device for which firmware is being loaded + * + * This function is similar in behaviour to request_firmware(), except it + * doesn't produce warning messages when the file is not found. The sysfs + * fallback mechanism is enabled if direct filesystem lookup fails. However, + * failures to find the firmware file with it are still suppressed. It is + * therefore up to the driver to check for the return value of this call and to + * decide when to inform the users of errors. + **/ +int firmware_request_nowarn(const struct firmware **firmware, const char *name, + struct device *device) +{ + int ret; + + /* Need to pin this module until return */ + __module_get(THIS_MODULE); + ret = _request_firmware(firmware, name, device, NULL, 0, 0, + FW_OPT_UEVENT | FW_OPT_NO_WARN); + module_put(THIS_MODULE); + return ret; +} +EXPORT_SYMBOL_GPL(firmware_request_nowarn); + +/** + * request_firmware_direct() - load firmware directly without usermode helper + * @firmware_p: pointer to firmware image + * @name: name of firmware file + * @device: device for which firmware is being loaded + * + * This function works pretty much like request_firmware(), but this doesn't + * fall back to usermode helper even if the firmware couldn't be loaded + * directly from fs. Hence it's useful for loading optional firmwares, which + * aren't always present, without extra long timeouts of udev. + **/ +int request_firmware_direct(const struct firmware **firmware_p, + const char *name, struct device *device) +{ + int ret; + + __module_get(THIS_MODULE); + ret = _request_firmware(firmware_p, name, device, NULL, 0, 0, + FW_OPT_UEVENT | FW_OPT_NO_WARN | + FW_OPT_NOFALLBACK_SYSFS); + module_put(THIS_MODULE); + return ret; +} +EXPORT_SYMBOL_GPL(request_firmware_direct); + +/** + * firmware_request_platform() - request firmware with platform-fw fallback + * @firmware: pointer to firmware image + * @name: name of firmware file + * @device: device for which firmware is being loaded + * + * This function is similar in behaviour to request_firmware, except that if + * direct filesystem lookup fails, it will fallback to looking for a copy of the + * requested firmware embedded in the platform's main (e.g. UEFI) firmware. + **/ +int firmware_request_platform(const struct firmware **firmware, + const char *name, struct device *device) +{ + int ret; + + /* Need to pin this module until return */ + __module_get(THIS_MODULE); + ret = _request_firmware(firmware, name, device, NULL, 0, 0, + FW_OPT_UEVENT | FW_OPT_FALLBACK_PLATFORM); + module_put(THIS_MODULE); + return ret; +} +EXPORT_SYMBOL_GPL(firmware_request_platform); + +/** + * firmware_request_cache() - cache firmware for suspend so resume can use it + * @device: device for which firmware should be cached for + * @name: name of firmware file + * + * There are some devices with an optimization that enables the device to not + * require loading firmware on system reboot. This optimization may still + * require the firmware present on resume from suspend. This routine can be + * used to ensure the firmware is present on resume from suspend in these + * situations. This helper is not compatible with drivers which use + * request_firmware_into_buf() or request_firmware_nowait() with no uevent set. + **/ +int firmware_request_cache(struct device *device, const char *name) +{ + int ret; + + mutex_lock(&fw_lock); + ret = fw_add_devm_name(device, name); + mutex_unlock(&fw_lock); + + return ret; +} +EXPORT_SYMBOL_GPL(firmware_request_cache); + +/** + * request_firmware_into_buf() - load firmware into a previously allocated buffer + * @firmware_p: pointer to firmware image + * @name: name of firmware file + * @device: device for which firmware is being loaded and DMA region allocated + * @buf: address of buffer to load firmware into + * @size: size of buffer + * + * This function works pretty much like request_firmware(), but it doesn't + * allocate a buffer to hold the firmware data. Instead, the firmware + * is loaded directly into the buffer pointed to by @buf and the @firmware_p + * data member is pointed at @buf. + * + * This function doesn't cache firmware either. + */ +int +request_firmware_into_buf(const struct firmware **firmware_p, const char *name, + struct device *device, void *buf, size_t size) +{ + int ret; + + if (fw_cache_is_setup(device, name)) + return -EOPNOTSUPP; + + __module_get(THIS_MODULE); + ret = _request_firmware(firmware_p, name, device, buf, size, 0, + FW_OPT_UEVENT | FW_OPT_NOCACHE); + module_put(THIS_MODULE); + return ret; +} +EXPORT_SYMBOL(request_firmware_into_buf); + +/** + * request_partial_firmware_into_buf() - load partial firmware into a previously allocated buffer + * @firmware_p: pointer to firmware image + * @name: name of firmware file + * @device: device for which firmware is being loaded and DMA region allocated + * @buf: address of buffer to load firmware into + * @size: size of buffer + * @offset: offset into file to read + * + * This function works pretty much like request_firmware_into_buf except + * it allows a partial read of the file. + */ +int +request_partial_firmware_into_buf(const struct firmware **firmware_p, + const char *name, struct device *device, + void *buf, size_t size, size_t offset) +{ + int ret; + + if (fw_cache_is_setup(device, name)) + return -EOPNOTSUPP; + + __module_get(THIS_MODULE); + ret = _request_firmware(firmware_p, name, device, buf, size, offset, + FW_OPT_UEVENT | FW_OPT_NOCACHE | + FW_OPT_PARTIAL); + module_put(THIS_MODULE); + return ret; +} +EXPORT_SYMBOL(request_partial_firmware_into_buf); + +/** + * release_firmware() - release the resource associated with a firmware image + * @fw: firmware resource to release + **/ +void release_firmware(const struct firmware *fw) +{ + if (fw) { + if (!firmware_is_builtin(fw)) + firmware_free_data(fw); + kfree(fw); + } +} +EXPORT_SYMBOL(release_firmware); + +/* Async support */ +struct firmware_work { + struct work_struct work; + struct module *module; + const char *name; + struct device *device; + void *context; + void (*cont)(const struct firmware *fw, void *context); + u32 opt_flags; +}; + +static void request_firmware_work_func(struct work_struct *work) +{ + struct firmware_work *fw_work; + const struct firmware *fw; + + fw_work = container_of(work, struct firmware_work, work); + + _request_firmware(&fw, fw_work->name, fw_work->device, NULL, 0, 0, + fw_work->opt_flags); + fw_work->cont(fw, fw_work->context); + put_device(fw_work->device); /* taken in request_firmware_nowait() */ + + module_put(fw_work->module); + kfree_const(fw_work->name); + kfree(fw_work); +} + + +static int _request_firmware_nowait( + struct module *module, bool uevent, + const char *name, struct device *device, gfp_t gfp, void *context, + void (*cont)(const struct firmware *fw, void *context), bool nowarn) +{ + struct firmware_work *fw_work; + + fw_work = kzalloc(sizeof(struct firmware_work), gfp); + if (!fw_work) + return -ENOMEM; + + fw_work->module = module; + fw_work->name = kstrdup_const(name, gfp); + if (!fw_work->name) { + kfree(fw_work); + return -ENOMEM; + } + fw_work->device = device; + fw_work->context = context; + fw_work->cont = cont; + fw_work->opt_flags = FW_OPT_NOWAIT | + (uevent ? FW_OPT_UEVENT : FW_OPT_USERHELPER) | + (nowarn ? FW_OPT_NO_WARN : 0); + + if (!uevent && fw_cache_is_setup(device, name)) { + kfree_const(fw_work->name); + kfree(fw_work); + return -EOPNOTSUPP; + } + + if (!try_module_get(module)) { + kfree_const(fw_work->name); + kfree(fw_work); + return -EFAULT; + } + + get_device(fw_work->device); + INIT_WORK(&fw_work->work, request_firmware_work_func); + schedule_work(&fw_work->work); + return 0; +} + +/** + * request_firmware_nowait() - asynchronous version of request_firmware + * @module: module requesting the firmware + * @uevent: sends uevent to copy the firmware image if this flag + * is non-zero else the firmware copy must be done manually. + * @name: name of firmware file + * @device: device for which firmware is being loaded + * @gfp: allocation flags + * @context: will be passed over to @cont, and + * @fw may be %NULL if firmware request fails. + * @cont: function will be called asynchronously when the firmware + * request is over. + * + * Caller must hold the reference count of @device. + * + * Asynchronous variant of request_firmware() for user contexts: + * - sleep for as small periods as possible since it may + * increase kernel boot time of built-in device drivers + * requesting firmware in their ->probe() methods, if + * @gfp is GFP_KERNEL. + * + * - can't sleep at all if @gfp is GFP_ATOMIC. + **/ +int request_firmware_nowait( + struct module *module, bool uevent, + const char *name, struct device *device, gfp_t gfp, void *context, + void (*cont)(const struct firmware *fw, void *context)) +{ + return _request_firmware_nowait(module, uevent, name, device, gfp, + context, cont, false); + +} +EXPORT_SYMBOL(request_firmware_nowait); + +/** + * firmware_request_nowait_nowarn() - async version of request_firmware_nowarn + * @module: module requesting the firmware + * @name: name of firmware file + * @device: device for which firmware is being loaded + * @gfp: allocation flags + * @context: will be passed over to @cont, and + * @fw may be %NULL if firmware request fails. + * @cont: function will be called asynchronously when the firmware + * request is over. + * + * Similar in function to request_firmware_nowait(), but doesn't print a warning + * when the firmware file could not be found and always sends a uevent to copy + * the firmware image. + */ +int firmware_request_nowait_nowarn( + struct module *module, const char *name, + struct device *device, gfp_t gfp, void *context, + void (*cont)(const struct firmware *fw, void *context)) +{ + return _request_firmware_nowait(module, FW_ACTION_UEVENT, name, device, + gfp, context, cont, true); +} +EXPORT_SYMBOL_GPL(firmware_request_nowait_nowarn); + +#ifdef CONFIG_FW_CACHE +static ASYNC_DOMAIN_EXCLUSIVE(fw_cache_domain); + +/** + * cache_firmware() - cache one firmware image in kernel memory space + * @fw_name: the firmware image name + * + * Cache firmware in kernel memory so that drivers can use it when + * system isn't ready for them to request firmware image from userspace. + * Once it returns successfully, driver can use request_firmware or its + * nowait version to get the cached firmware without any interacting + * with userspace + * + * Return 0 if the firmware image has been cached successfully + * Return !0 otherwise + * + */ +static int cache_firmware(const char *fw_name) +{ + int ret; + const struct firmware *fw; + + pr_debug("%s: %s\n", __func__, fw_name); + + ret = request_firmware(&fw, fw_name, NULL); + if (!ret) + kfree(fw); + + pr_debug("%s: %s ret=%d\n", __func__, fw_name, ret); + + return ret; +} + +static struct fw_priv *lookup_fw_priv(const char *fw_name) +{ + struct fw_priv *tmp; + struct firmware_cache *fwc = &fw_cache; + + spin_lock(&fwc->lock); + tmp = __lookup_fw_priv(fw_name); + spin_unlock(&fwc->lock); + + return tmp; +} + +/** + * uncache_firmware() - remove one cached firmware image + * @fw_name: the firmware image name + * + * Uncache one firmware image which has been cached successfully + * before. + * + * Return 0 if the firmware cache has been removed successfully + * Return !0 otherwise + * + */ +static int uncache_firmware(const char *fw_name) +{ + struct fw_priv *fw_priv; + struct firmware fw; + + pr_debug("%s: %s\n", __func__, fw_name); + + if (firmware_request_builtin(&fw, fw_name)) + return 0; + + fw_priv = lookup_fw_priv(fw_name); + if (fw_priv) { + free_fw_priv(fw_priv); + return 0; + } + + return -EINVAL; +} + +static struct fw_cache_entry *alloc_fw_cache_entry(const char *name) +{ + struct fw_cache_entry *fce; + + fce = kzalloc(sizeof(*fce), GFP_ATOMIC); + if (!fce) + goto exit; + + fce->name = kstrdup_const(name, GFP_ATOMIC); + if (!fce->name) { + kfree(fce); + fce = NULL; + goto exit; + } +exit: + return fce; +} + +static int __fw_entry_found(const char *name) +{ + struct firmware_cache *fwc = &fw_cache; + struct fw_cache_entry *fce; + + list_for_each_entry(fce, &fwc->fw_names, list) { + if (!strcmp(fce->name, name)) + return 1; + } + return 0; +} + +static void fw_cache_piggyback_on_request(struct fw_priv *fw_priv) +{ + const char *name = fw_priv->fw_name; + struct firmware_cache *fwc = fw_priv->fwc; + struct fw_cache_entry *fce; + + spin_lock(&fwc->name_lock); + if (__fw_entry_found(name)) + goto found; + + fce = alloc_fw_cache_entry(name); + if (fce) { + list_add(&fce->list, &fwc->fw_names); + kref_get(&fw_priv->ref); + pr_debug("%s: fw: %s\n", __func__, name); + } +found: + spin_unlock(&fwc->name_lock); +} + +static void free_fw_cache_entry(struct fw_cache_entry *fce) +{ + kfree_const(fce->name); + kfree(fce); +} + +static void __async_dev_cache_fw_image(void *fw_entry, + async_cookie_t cookie) +{ + struct fw_cache_entry *fce = fw_entry; + struct firmware_cache *fwc = &fw_cache; + int ret; + + ret = cache_firmware(fce->name); + if (ret) { + spin_lock(&fwc->name_lock); + list_del(&fce->list); + spin_unlock(&fwc->name_lock); + + free_fw_cache_entry(fce); + } +} + +/* called with dev->devres_lock held */ +static void dev_create_fw_entry(struct device *dev, void *res, + void *data) +{ + struct fw_name_devm *fwn = res; + const char *fw_name = fwn->name; + struct list_head *head = data; + struct fw_cache_entry *fce; + + fce = alloc_fw_cache_entry(fw_name); + if (fce) + list_add(&fce->list, head); +} + +static int devm_name_match(struct device *dev, void *res, + void *match_data) +{ + struct fw_name_devm *fwn = res; + return (fwn->magic == (unsigned long)match_data); +} + +static void dev_cache_fw_image(struct device *dev, void *data) +{ + LIST_HEAD(todo); + struct fw_cache_entry *fce; + struct fw_cache_entry *fce_next; + struct firmware_cache *fwc = &fw_cache; + + devres_for_each_res(dev, fw_name_devm_release, + devm_name_match, &fw_cache, + dev_create_fw_entry, &todo); + + list_for_each_entry_safe(fce, fce_next, &todo, list) { + list_del(&fce->list); + + spin_lock(&fwc->name_lock); + /* only one cache entry for one firmware */ + if (!__fw_entry_found(fce->name)) { + list_add(&fce->list, &fwc->fw_names); + } else { + free_fw_cache_entry(fce); + fce = NULL; + } + spin_unlock(&fwc->name_lock); + + if (fce) + async_schedule_domain(__async_dev_cache_fw_image, + (void *)fce, + &fw_cache_domain); + } +} + +static void __device_uncache_fw_images(void) +{ + struct firmware_cache *fwc = &fw_cache; + struct fw_cache_entry *fce; + + spin_lock(&fwc->name_lock); + while (!list_empty(&fwc->fw_names)) { + fce = list_entry(fwc->fw_names.next, + struct fw_cache_entry, list); + list_del(&fce->list); + spin_unlock(&fwc->name_lock); + + uncache_firmware(fce->name); + free_fw_cache_entry(fce); + + spin_lock(&fwc->name_lock); + } + spin_unlock(&fwc->name_lock); +} + +/** + * device_cache_fw_images() - cache devices' firmware + * + * If one device called request_firmware or its nowait version + * successfully before, the firmware names are recored into the + * device's devres link list, so device_cache_fw_images can call + * cache_firmware() to cache these firmwares for the device, + * then the device driver can load its firmwares easily at + * time when system is not ready to complete loading firmware. + */ +static void device_cache_fw_images(void) +{ + struct firmware_cache *fwc = &fw_cache; + DEFINE_WAIT(wait); + + pr_debug("%s\n", __func__); + + /* cancel uncache work */ + cancel_delayed_work_sync(&fwc->work); + + fw_fallback_set_cache_timeout(); + + mutex_lock(&fw_lock); + fwc->state = FW_LOADER_START_CACHE; + dpm_for_each_dev(NULL, dev_cache_fw_image); + mutex_unlock(&fw_lock); + + /* wait for completion of caching firmware for all devices */ + async_synchronize_full_domain(&fw_cache_domain); + + fw_fallback_set_default_timeout(); +} + +/** + * device_uncache_fw_images() - uncache devices' firmware + * + * uncache all firmwares which have been cached successfully + * by device_uncache_fw_images earlier + */ +static void device_uncache_fw_images(void) +{ + pr_debug("%s\n", __func__); + __device_uncache_fw_images(); +} + +static void device_uncache_fw_images_work(struct work_struct *work) +{ + device_uncache_fw_images(); +} + +/** + * device_uncache_fw_images_delay() - uncache devices firmwares + * @delay: number of milliseconds to delay uncache device firmwares + * + * uncache all devices's firmwares which has been cached successfully + * by device_cache_fw_images after @delay milliseconds. + */ +static void device_uncache_fw_images_delay(unsigned long delay) +{ + queue_delayed_work(system_power_efficient_wq, &fw_cache.work, + msecs_to_jiffies(delay)); +} + +static int fw_pm_notify(struct notifier_block *notify_block, + unsigned long mode, void *unused) +{ + switch (mode) { + case PM_HIBERNATION_PREPARE: + case PM_SUSPEND_PREPARE: + case PM_RESTORE_PREPARE: + /* + * Here, kill pending fallback requests will only kill + * non-uevent firmware request to avoid stalling suspend. + */ + kill_pending_fw_fallback_reqs(false); + device_cache_fw_images(); + break; + + case PM_POST_SUSPEND: + case PM_POST_HIBERNATION: + case PM_POST_RESTORE: + /* + * In case that system sleep failed and syscore_suspend is + * not called. + */ + mutex_lock(&fw_lock); + fw_cache.state = FW_LOADER_NO_CACHE; + mutex_unlock(&fw_lock); + + device_uncache_fw_images_delay(10 * MSEC_PER_SEC); + break; + } + + return 0; +} + +/* stop caching firmware once syscore_suspend is reached */ +static int fw_suspend(void *data) +{ + fw_cache.state = FW_LOADER_NO_CACHE; + return 0; +} + +static const struct syscore_ops fw_syscore_ops = { + .suspend = fw_suspend, +}; + +static struct syscore fw_syscore = { + .ops = &fw_syscore_ops, +}; + +static int __init register_fw_pm_ops(void) +{ + int ret; + + spin_lock_init(&fw_cache.name_lock); + INIT_LIST_HEAD(&fw_cache.fw_names); + + INIT_DELAYED_WORK(&fw_cache.work, + device_uncache_fw_images_work); + + fw_cache.pm_notify.notifier_call = fw_pm_notify; + ret = register_pm_notifier(&fw_cache.pm_notify); + if (ret) + return ret; + + register_syscore(&fw_syscore); + + return ret; +} + +static inline void unregister_fw_pm_ops(void) +{ + unregister_syscore(&fw_syscore); + unregister_pm_notifier(&fw_cache.pm_notify); +} +#else +static void fw_cache_piggyback_on_request(struct fw_priv *fw_priv) +{ +} +static inline int register_fw_pm_ops(void) +{ + return 0; +} +static inline void unregister_fw_pm_ops(void) +{ +} +#endif + +static void __init fw_cache_init(void) +{ + spin_lock_init(&fw_cache.lock); + INIT_LIST_HEAD(&fw_cache.head); + fw_cache.state = FW_LOADER_NO_CACHE; +} + +static int fw_shutdown_notify(struct notifier_block *unused1, + unsigned long unused2, void *unused3) +{ + /* + * Kill all pending fallback requests to avoid both stalling shutdown, + * and avoid a deadlock with the usermode_lock. + */ + kill_pending_fw_fallback_reqs(true); + + return NOTIFY_DONE; +} + +static struct notifier_block fw_shutdown_nb = { + .notifier_call = fw_shutdown_notify, +}; + +static int __init firmware_class_init(void) +{ + int ret; + + /* No need to unfold these on exit */ + fw_cache_init(); + + ret = register_fw_pm_ops(); + if (ret) + return ret; + + ret = register_reboot_notifier(&fw_shutdown_nb); + if (ret) + goto out; + + return register_sysfs_loader(); + +out: + unregister_fw_pm_ops(); + return ret; +} + +static void __exit firmware_class_exit(void) +{ + unregister_fw_pm_ops(); + unregister_reboot_notifier(&fw_shutdown_nb); + unregister_sysfs_loader(); +} + +fs_initcall(firmware_class_init); +module_exit(firmware_class_exit); diff --git a/drivers/base/firmware_loader/sysfs.c b/drivers/base/firmware_loader/sysfs.c new file mode 100644 index 000000000000..92e91050f96a --- /dev/null +++ b/drivers/base/firmware_loader/sysfs.c @@ -0,0 +1,425 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/highmem.h> +#include <linux/module.h> +#include <linux/security.h> +#include <linux/slab.h> +#include <linux/types.h> + +#include "sysfs.h" + +/* + * sysfs support for firmware loader + */ + +void __fw_load_abort(struct fw_priv *fw_priv) +{ + /* + * There is a small window in which user can write to 'loading' + * between loading done/aborted and disappearance of 'loading' + */ + if (fw_state_is_aborted(fw_priv) || fw_state_is_done(fw_priv)) + return; + + fw_state_aborted(fw_priv); +} + +#ifdef CONFIG_FW_LOADER_USER_HELPER +static ssize_t timeout_show(const struct class *class, const struct class_attribute *attr, + char *buf) +{ + return sysfs_emit(buf, "%d\n", __firmware_loading_timeout()); +} + +/** + * timeout_store() - set number of seconds to wait for firmware + * @class: device class pointer + * @attr: device attribute pointer + * @buf: buffer to scan for timeout value + * @count: number of bytes in @buf + * + * Sets the number of seconds to wait for the firmware. Once + * this expires an error will be returned to the driver and no + * firmware will be provided. + * + * Note: zero means 'wait forever'. + **/ +static ssize_t timeout_store(const struct class *class, const struct class_attribute *attr, + const char *buf, size_t count) +{ + int tmp_loading_timeout; + + if (kstrtoint(buf, 10, &tmp_loading_timeout)) + return -EINVAL; + + if (tmp_loading_timeout < 0) + tmp_loading_timeout = 0; + + __fw_fallback_set_timeout(tmp_loading_timeout); + + return count; +} +static CLASS_ATTR_RW(timeout); + +static struct attribute *firmware_class_attrs[] = { + &class_attr_timeout.attr, + NULL, +}; +ATTRIBUTE_GROUPS(firmware_class); + +static int do_firmware_uevent(const struct fw_sysfs *fw_sysfs, struct kobj_uevent_env *env) +{ + if (add_uevent_var(env, "FIRMWARE=%s", fw_sysfs->fw_priv->fw_name)) + return -ENOMEM; + if (add_uevent_var(env, "TIMEOUT=%i", __firmware_loading_timeout())) + return -ENOMEM; + if (add_uevent_var(env, "ASYNC=%d", fw_sysfs->nowait)) + return -ENOMEM; + + return 0; +} + +static int firmware_uevent(const struct device *dev, struct kobj_uevent_env *env) +{ + const struct fw_sysfs *fw_sysfs = to_fw_sysfs(dev); + int err = 0; + + mutex_lock(&fw_lock); + if (fw_sysfs->fw_priv) + err = do_firmware_uevent(fw_sysfs, env); + mutex_unlock(&fw_lock); + return err; +} +#endif /* CONFIG_FW_LOADER_USER_HELPER */ + +static void fw_dev_release(struct device *dev) +{ + struct fw_sysfs *fw_sysfs = to_fw_sysfs(dev); + + if (fw_sysfs->fw_upload_priv) + fw_upload_free(fw_sysfs); + + kfree(fw_sysfs); +} + +static struct class firmware_class = { + .name = "firmware", +#ifdef CONFIG_FW_LOADER_USER_HELPER + .class_groups = firmware_class_groups, + .dev_uevent = firmware_uevent, +#endif + .dev_release = fw_dev_release, +}; + +int register_sysfs_loader(void) +{ + int ret = class_register(&firmware_class); + + if (ret != 0) + return ret; + return register_firmware_config_sysctl(); +} + +void unregister_sysfs_loader(void) +{ + unregister_firmware_config_sysctl(); + class_unregister(&firmware_class); +} + +static ssize_t firmware_loading_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct fw_sysfs *fw_sysfs = to_fw_sysfs(dev); + int loading = 0; + + mutex_lock(&fw_lock); + if (fw_sysfs->fw_priv) + loading = fw_state_is_loading(fw_sysfs->fw_priv); + mutex_unlock(&fw_lock); + + return sysfs_emit(buf, "%d\n", loading); +} + +/** + * firmware_loading_store() - set value in the 'loading' control file + * @dev: device pointer + * @attr: device attribute pointer + * @buf: buffer to scan for loading control value + * @count: number of bytes in @buf + * + * The relevant values are: + * + * 1: Start a load, discarding any previous partial load. + * 0: Conclude the load and hand the data to the driver code. + * -1: Conclude the load with an error and discard any written data. + **/ +static ssize_t firmware_loading_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct fw_sysfs *fw_sysfs = to_fw_sysfs(dev); + struct fw_priv *fw_priv; + ssize_t written = count; + int loading; + + if (kstrtoint(buf, 10, &loading)) + return -EINVAL; + + mutex_lock(&fw_lock); + fw_priv = fw_sysfs->fw_priv; + if (fw_state_is_aborted(fw_priv) || fw_state_is_done(fw_priv)) + goto out; + + switch (loading) { + case 1: + /* discarding any previous partial load */ + fw_free_paged_buf(fw_priv); + fw_state_start(fw_priv); + break; + case 0: + if (fw_state_is_loading(fw_priv)) { + int rc; + + /* + * Several loading requests may be pending on + * one same firmware buf, so let all requests + * see the mapped 'buf->data' once the loading + * is completed. + */ + rc = fw_map_paged_buf(fw_priv); + if (rc) + dev_err(dev, "%s: map pages failed\n", + __func__); + else + rc = security_kernel_post_load_data(fw_priv->data, + fw_priv->size, + LOADING_FIRMWARE, + "blob"); + + /* + * Same logic as fw_load_abort, only the DONE bit + * is ignored and we set ABORT only on failure. + */ + if (rc) { + fw_state_aborted(fw_priv); + written = rc; + } else { + fw_state_done(fw_priv); + + /* + * If this is a user-initiated firmware upload + * then start the upload in a worker thread now. + */ + rc = fw_upload_start(fw_sysfs); + if (rc) + written = rc; + } + break; + } + fallthrough; + default: + dev_err(dev, "%s: unexpected value (%d)\n", __func__, loading); + fallthrough; + case -1: + fw_load_abort(fw_sysfs); + if (fw_sysfs->fw_upload_priv) + fw_state_init(fw_sysfs->fw_priv); + + break; + } +out: + mutex_unlock(&fw_lock); + return written; +} + +DEVICE_ATTR(loading, 0644, firmware_loading_show, firmware_loading_store); + +static void firmware_rw_data(struct fw_priv *fw_priv, char *buffer, + loff_t offset, size_t count, bool read) +{ + if (read) + memcpy(buffer, fw_priv->data + offset, count); + else + memcpy(fw_priv->data + offset, buffer, count); +} + +static void firmware_rw(struct fw_priv *fw_priv, char *buffer, + loff_t offset, size_t count, bool read) +{ + while (count) { + int page_nr = offset >> PAGE_SHIFT; + int page_ofs = offset & (PAGE_SIZE - 1); + int page_cnt = min_t(size_t, PAGE_SIZE - page_ofs, count); + + if (read) + memcpy_from_page(buffer, fw_priv->pages[page_nr], + page_ofs, page_cnt); + else + memcpy_to_page(fw_priv->pages[page_nr], page_ofs, + buffer, page_cnt); + + buffer += page_cnt; + offset += page_cnt; + count -= page_cnt; + } +} + +static ssize_t firmware_data_read(struct file *filp, struct kobject *kobj, + const struct bin_attribute *bin_attr, + char *buffer, loff_t offset, size_t count) +{ + struct device *dev = kobj_to_dev(kobj); + struct fw_sysfs *fw_sysfs = to_fw_sysfs(dev); + struct fw_priv *fw_priv; + ssize_t ret_count; + + mutex_lock(&fw_lock); + fw_priv = fw_sysfs->fw_priv; + if (!fw_priv || fw_state_is_done(fw_priv)) { + ret_count = -ENODEV; + goto out; + } + if (offset > fw_priv->size) { + ret_count = 0; + goto out; + } + if (count > fw_priv->size - offset) + count = fw_priv->size - offset; + + ret_count = count; + + if (fw_priv->data) + firmware_rw_data(fw_priv, buffer, offset, count, true); + else + firmware_rw(fw_priv, buffer, offset, count, true); + +out: + mutex_unlock(&fw_lock); + return ret_count; +} + +static int fw_realloc_pages(struct fw_sysfs *fw_sysfs, int min_size) +{ + int err; + + err = fw_grow_paged_buf(fw_sysfs->fw_priv, + PAGE_ALIGN(min_size) >> PAGE_SHIFT); + if (err) + fw_load_abort(fw_sysfs); + return err; +} + +/** + * firmware_data_write() - write method for firmware + * @filp: open sysfs file + * @kobj: kobject for the device + * @bin_attr: bin_attr structure + * @buffer: buffer being written + * @offset: buffer offset for write in total data store area + * @count: buffer size + * + * Data written to the 'data' attribute will be later handed to + * the driver as a firmware image. + **/ +static ssize_t firmware_data_write(struct file *filp, struct kobject *kobj, + const struct bin_attribute *bin_attr, + char *buffer, loff_t offset, size_t count) +{ + struct device *dev = kobj_to_dev(kobj); + struct fw_sysfs *fw_sysfs = to_fw_sysfs(dev); + struct fw_priv *fw_priv; + ssize_t retval; + + if (!capable(CAP_SYS_RAWIO)) + return -EPERM; + + mutex_lock(&fw_lock); + fw_priv = fw_sysfs->fw_priv; + if (!fw_priv || fw_state_is_done(fw_priv)) { + retval = -ENODEV; + goto out; + } + + if (fw_priv->data) { + if (offset + count > fw_priv->allocated_size) { + retval = -ENOMEM; + goto out; + } + firmware_rw_data(fw_priv, buffer, offset, count, false); + retval = count; + } else { + retval = fw_realloc_pages(fw_sysfs, offset + count); + if (retval) + goto out; + + retval = count; + firmware_rw(fw_priv, buffer, offset, count, false); + } + + fw_priv->size = max_t(size_t, offset + count, fw_priv->size); +out: + mutex_unlock(&fw_lock); + return retval; +} + +static const struct bin_attribute firmware_attr_data = { + .attr = { .name = "data", .mode = 0644 }, + .size = 0, + .read = firmware_data_read, + .write = firmware_data_write, +}; + +static struct attribute *fw_dev_attrs[] = { + &dev_attr_loading.attr, +#ifdef CONFIG_FW_UPLOAD + &dev_attr_cancel.attr, + &dev_attr_status.attr, + &dev_attr_error.attr, + &dev_attr_remaining_size.attr, +#endif + NULL +}; + +static const struct bin_attribute *const fw_dev_bin_attrs[] = { + &firmware_attr_data, + NULL +}; + +static const struct attribute_group fw_dev_attr_group = { + .attrs = fw_dev_attrs, + .bin_attrs = fw_dev_bin_attrs, +#ifdef CONFIG_FW_UPLOAD + .is_visible = fw_upload_is_visible, +#endif +}; + +static const struct attribute_group *fw_dev_attr_groups[] = { + &fw_dev_attr_group, + NULL +}; + +struct fw_sysfs * +fw_create_instance(struct firmware *firmware, const char *fw_name, + struct device *device, u32 opt_flags) +{ + struct fw_sysfs *fw_sysfs; + struct device *f_dev; + + fw_sysfs = kzalloc(sizeof(*fw_sysfs), GFP_KERNEL); + if (!fw_sysfs) { + fw_sysfs = ERR_PTR(-ENOMEM); + goto exit; + } + + fw_sysfs->nowait = !!(opt_flags & FW_OPT_NOWAIT); + fw_sysfs->fw = firmware; + f_dev = &fw_sysfs->dev; + + device_initialize(f_dev); + dev_set_name(f_dev, "%s", fw_name); + f_dev->parent = device; + f_dev->class = &firmware_class; + f_dev->groups = fw_dev_attr_groups; +exit: + return fw_sysfs; +} diff --git a/drivers/base/firmware_loader/sysfs.h b/drivers/base/firmware_loader/sysfs.h new file mode 100644 index 000000000000..1cabea544a40 --- /dev/null +++ b/drivers/base/firmware_loader/sysfs.h @@ -0,0 +1,118 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __FIRMWARE_SYSFS_H +#define __FIRMWARE_SYSFS_H + +#include <linux/device.h> + +#include "firmware.h" + +MODULE_IMPORT_NS("FIRMWARE_LOADER_PRIVATE"); + +extern struct firmware_fallback_config fw_fallback_config; +extern struct device_attribute dev_attr_loading; + +#ifdef CONFIG_FW_LOADER_USER_HELPER +/** + * struct firmware_fallback_config - firmware fallback configuration settings + * + * Helps describe and fine tune the fallback mechanism. + * + * @force_sysfs_fallback: force the sysfs fallback mechanism to be used + * as if one had enabled CONFIG_FW_LOADER_USER_HELPER_FALLBACK=y. + * Useful to help debug a CONFIG_FW_LOADER_USER_HELPER_FALLBACK=y + * functionality on a kernel where that config entry has been disabled. + * @ignore_sysfs_fallback: force to disable the sysfs fallback mechanism. + * This emulates the behaviour as if we had set the kernel + * config CONFIG_FW_LOADER_USER_HELPER=n. + * @old_timeout: for internal use + * @loading_timeout: the timeout to wait for the fallback mechanism before + * giving up, in seconds. + */ +struct firmware_fallback_config { + unsigned int force_sysfs_fallback; + unsigned int ignore_sysfs_fallback; + int old_timeout; + int loading_timeout; +}; + +/* These getters are vetted to use int properly */ +static inline int __firmware_loading_timeout(void) +{ + return fw_fallback_config.loading_timeout; +} + +/* These setters are vetted to use int properly */ +static inline void __fw_fallback_set_timeout(int timeout) +{ + fw_fallback_config.loading_timeout = timeout; +} +#endif + +#ifdef CONFIG_FW_LOADER_SYSFS +int register_sysfs_loader(void); +void unregister_sysfs_loader(void); +#if defined(CONFIG_FW_LOADER_USER_HELPER) && defined(CONFIG_SYSCTL) +int register_firmware_config_sysctl(void); +void unregister_firmware_config_sysctl(void); +#else +static inline int register_firmware_config_sysctl(void) +{ + return 0; +} + +static inline void unregister_firmware_config_sysctl(void) { } +#endif /* CONFIG_FW_LOADER_USER_HELPER && CONFIG_SYSCTL */ +#else /* CONFIG_FW_LOADER_SYSFS */ +static inline int register_sysfs_loader(void) +{ + return 0; +} + +static inline void unregister_sysfs_loader(void) +{ +} +#endif /* CONFIG_FW_LOADER_SYSFS */ + +struct fw_sysfs { + bool nowait; + struct device dev; + struct fw_priv *fw_priv; + struct firmware *fw; + void *fw_upload_priv; +}; +#define to_fw_sysfs(__dev) container_of_const(__dev, struct fw_sysfs, dev) + +void __fw_load_abort(struct fw_priv *fw_priv); + +static inline void fw_load_abort(struct fw_sysfs *fw_sysfs) +{ + struct fw_priv *fw_priv = fw_sysfs->fw_priv; + + __fw_load_abort(fw_priv); +} + +struct fw_sysfs * +fw_create_instance(struct firmware *firmware, const char *fw_name, + struct device *device, u32 opt_flags); + +#ifdef CONFIG_FW_UPLOAD +extern struct device_attribute dev_attr_status; +extern struct device_attribute dev_attr_error; +extern struct device_attribute dev_attr_cancel; +extern struct device_attribute dev_attr_remaining_size; + +int fw_upload_start(struct fw_sysfs *fw_sysfs); +void fw_upload_free(struct fw_sysfs *fw_sysfs); +umode_t fw_upload_is_visible(struct kobject *kobj, struct attribute *attr, int n); +#else +static inline int fw_upload_start(struct fw_sysfs *fw_sysfs) +{ + return 0; +} + +static inline void fw_upload_free(struct fw_sysfs *fw_sysfs) +{ +} +#endif + +#endif /* __FIRMWARE_SYSFS_H */ diff --git a/drivers/base/firmware_loader/sysfs_upload.c b/drivers/base/firmware_loader/sysfs_upload.c new file mode 100644 index 000000000000..c3797b93c5f5 --- /dev/null +++ b/drivers/base/firmware_loader/sysfs_upload.c @@ -0,0 +1,410 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/firmware.h> +#include <linux/module.h> +#include <linux/slab.h> + +#include "sysfs_upload.h" + +/* + * Support for user-space to initiate a firmware upload to a device. + */ + +static const char * const fw_upload_prog_str[] = { + [FW_UPLOAD_PROG_IDLE] = "idle", + [FW_UPLOAD_PROG_RECEIVING] = "receiving", + [FW_UPLOAD_PROG_PREPARING] = "preparing", + [FW_UPLOAD_PROG_TRANSFERRING] = "transferring", + [FW_UPLOAD_PROG_PROGRAMMING] = "programming" +}; + +static const char * const fw_upload_err_str[] = { + [FW_UPLOAD_ERR_NONE] = "none", + [FW_UPLOAD_ERR_HW_ERROR] = "hw-error", + [FW_UPLOAD_ERR_TIMEOUT] = "timeout", + [FW_UPLOAD_ERR_CANCELED] = "user-abort", + [FW_UPLOAD_ERR_BUSY] = "device-busy", + [FW_UPLOAD_ERR_INVALID_SIZE] = "invalid-file-size", + [FW_UPLOAD_ERR_RW_ERROR] = "read-write-error", + [FW_UPLOAD_ERR_WEAROUT] = "flash-wearout", + [FW_UPLOAD_ERR_FW_INVALID] = "firmware-invalid", +}; + +static const char *fw_upload_progress(struct device *dev, + enum fw_upload_prog prog) +{ + const char *status = "unknown-status"; + + if (prog < FW_UPLOAD_PROG_MAX) + status = fw_upload_prog_str[prog]; + else + dev_err(dev, "Invalid status during secure update: %d\n", prog); + + return status; +} + +static const char *fw_upload_error(struct device *dev, + enum fw_upload_err err_code) +{ + const char *error = "unknown-error"; + + if (err_code < FW_UPLOAD_ERR_MAX) + error = fw_upload_err_str[err_code]; + else + dev_err(dev, "Invalid error code during secure update: %d\n", + err_code); + + return error; +} + +static ssize_t +status_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct fw_upload_priv *fwlp = to_fw_sysfs(dev)->fw_upload_priv; + + return sysfs_emit(buf, "%s\n", fw_upload_progress(dev, fwlp->progress)); +} +DEVICE_ATTR_RO(status); + +static ssize_t +error_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct fw_upload_priv *fwlp = to_fw_sysfs(dev)->fw_upload_priv; + int ret; + + mutex_lock(&fwlp->lock); + + if (fwlp->progress != FW_UPLOAD_PROG_IDLE) + ret = -EBUSY; + else if (!fwlp->err_code) + ret = 0; + else + ret = sysfs_emit(buf, "%s:%s\n", + fw_upload_progress(dev, fwlp->err_progress), + fw_upload_error(dev, fwlp->err_code)); + + mutex_unlock(&fwlp->lock); + + return ret; +} +DEVICE_ATTR_RO(error); + +static ssize_t cancel_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct fw_upload_priv *fwlp = to_fw_sysfs(dev)->fw_upload_priv; + int ret = count; + bool cancel; + + if (kstrtobool(buf, &cancel) || !cancel) + return -EINVAL; + + mutex_lock(&fwlp->lock); + if (fwlp->progress == FW_UPLOAD_PROG_IDLE) { + mutex_unlock(&fwlp->lock); + return -ENODEV; + } + + fwlp->ops->cancel(fwlp->fw_upload); + mutex_unlock(&fwlp->lock); + + return ret; +} +DEVICE_ATTR_WO(cancel); + +static ssize_t remaining_size_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct fw_upload_priv *fwlp = to_fw_sysfs(dev)->fw_upload_priv; + + return sysfs_emit(buf, "%u\n", fwlp->remaining_size); +} +DEVICE_ATTR_RO(remaining_size); + +umode_t +fw_upload_is_visible(struct kobject *kobj, struct attribute *attr, int n) +{ + static struct fw_sysfs *fw_sysfs; + + fw_sysfs = to_fw_sysfs(kobj_to_dev(kobj)); + + if (fw_sysfs->fw_upload_priv || attr == &dev_attr_loading.attr) + return attr->mode; + + return 0; +} + +static void fw_upload_update_progress(struct fw_upload_priv *fwlp, + enum fw_upload_prog new_progress) +{ + mutex_lock(&fwlp->lock); + fwlp->progress = new_progress; + mutex_unlock(&fwlp->lock); +} + +static void fw_upload_set_error(struct fw_upload_priv *fwlp, + enum fw_upload_err err_code) +{ + mutex_lock(&fwlp->lock); + fwlp->err_progress = fwlp->progress; + fwlp->err_code = err_code; + mutex_unlock(&fwlp->lock); +} + +static void fw_upload_prog_complete(struct fw_upload_priv *fwlp) +{ + mutex_lock(&fwlp->lock); + fwlp->progress = FW_UPLOAD_PROG_IDLE; + mutex_unlock(&fwlp->lock); +} + +static void fw_upload_main(struct work_struct *work) +{ + struct fw_upload_priv *fwlp; + struct fw_sysfs *fw_sysfs; + u32 written = 0, offset = 0; + enum fw_upload_err ret; + struct device *fw_dev; + struct fw_upload *fwl; + + fwlp = container_of(work, struct fw_upload_priv, work); + fwl = fwlp->fw_upload; + fw_sysfs = (struct fw_sysfs *)fwl->priv; + fw_dev = &fw_sysfs->dev; + + fw_upload_update_progress(fwlp, FW_UPLOAD_PROG_PREPARING); + ret = fwlp->ops->prepare(fwl, fwlp->data, fwlp->remaining_size); + if (ret != FW_UPLOAD_ERR_NONE) { + fw_upload_set_error(fwlp, ret); + goto putdev_exit; + } + + fw_upload_update_progress(fwlp, FW_UPLOAD_PROG_TRANSFERRING); + while (fwlp->remaining_size) { + ret = fwlp->ops->write(fwl, fwlp->data, offset, + fwlp->remaining_size, &written); + if (ret != FW_UPLOAD_ERR_NONE || !written) { + if (ret == FW_UPLOAD_ERR_NONE) { + dev_warn(fw_dev, "write-op wrote zero data\n"); + ret = FW_UPLOAD_ERR_RW_ERROR; + } + fw_upload_set_error(fwlp, ret); + goto done; + } + + fwlp->remaining_size -= written; + offset += written; + } + + fw_upload_update_progress(fwlp, FW_UPLOAD_PROG_PROGRAMMING); + ret = fwlp->ops->poll_complete(fwl); + if (ret != FW_UPLOAD_ERR_NONE) + fw_upload_set_error(fwlp, ret); + +done: + if (fwlp->ops->cleanup) + fwlp->ops->cleanup(fwl); + +putdev_exit: + put_device(fw_dev->parent); + + /* + * Note: fwlp->remaining_size is left unmodified here to provide + * additional information on errors. It will be reinitialized when + * the next firmeware upload begins. + */ + mutex_lock(&fw_lock); + fw_free_paged_buf(fw_sysfs->fw_priv); + fw_state_init(fw_sysfs->fw_priv); + mutex_unlock(&fw_lock); + fwlp->data = NULL; + fw_upload_prog_complete(fwlp); +} + +/* + * Start a worker thread to upload data to the parent driver. + * Must be called with fw_lock held. + */ +int fw_upload_start(struct fw_sysfs *fw_sysfs) +{ + struct fw_priv *fw_priv = fw_sysfs->fw_priv; + struct device *fw_dev = &fw_sysfs->dev; + struct fw_upload_priv *fwlp; + + if (!fw_sysfs->fw_upload_priv) + return 0; + + if (!fw_priv->size) { + fw_free_paged_buf(fw_priv); + fw_state_init(fw_sysfs->fw_priv); + return 0; + } + + fwlp = fw_sysfs->fw_upload_priv; + mutex_lock(&fwlp->lock); + + /* Do not interfere with an on-going fw_upload */ + if (fwlp->progress != FW_UPLOAD_PROG_IDLE) { + mutex_unlock(&fwlp->lock); + return -EBUSY; + } + + get_device(fw_dev->parent); /* released in fw_upload_main */ + + fwlp->progress = FW_UPLOAD_PROG_RECEIVING; + fwlp->err_code = 0; + fwlp->remaining_size = fw_priv->size; + fwlp->data = fw_priv->data; + + pr_debug("%s: fw-%s fw_priv=%p data=%p size=%u\n", + __func__, fw_priv->fw_name, + fw_priv, fw_priv->data, + (unsigned int)fw_priv->size); + + queue_work(system_long_wq, &fwlp->work); + mutex_unlock(&fwlp->lock); + + return 0; +} + +void fw_upload_free(struct fw_sysfs *fw_sysfs) +{ + struct fw_upload_priv *fw_upload_priv = fw_sysfs->fw_upload_priv; + + free_fw_priv(fw_sysfs->fw_priv); + kfree(fw_upload_priv->fw_upload); + kfree(fw_upload_priv); +} + +/** + * firmware_upload_register() - register for the firmware upload sysfs API + * @module: kernel module of this device + * @parent: parent device instantiating firmware upload + * @name: firmware name to be associated with this device + * @ops: pointer to structure of firmware upload ops + * @dd_handle: pointer to parent driver private data + * + * @name must be unique among all users of firmware upload. The firmware + * sysfs files for this device will be found at /sys/class/firmware/@name. + * + * Return: struct fw_upload pointer or ERR_PTR() + * + **/ +struct fw_upload * +firmware_upload_register(struct module *module, struct device *parent, + const char *name, const struct fw_upload_ops *ops, + void *dd_handle) +{ + u32 opt_flags = FW_OPT_NOCACHE; + struct fw_upload *fw_upload; + struct fw_upload_priv *fw_upload_priv; + struct fw_sysfs *fw_sysfs; + struct fw_priv *fw_priv; + struct device *fw_dev; + int ret; + + if (!name || name[0] == '\0') + return ERR_PTR(-EINVAL); + + if (!ops || !ops->cancel || !ops->prepare || + !ops->write || !ops->poll_complete) { + dev_err(parent, "Attempt to register without all required ops\n"); + return ERR_PTR(-EINVAL); + } + + if (!try_module_get(module)) + return ERR_PTR(-EFAULT); + + fw_upload = kzalloc(sizeof(*fw_upload), GFP_KERNEL); + if (!fw_upload) { + ret = -ENOMEM; + goto exit_module_put; + } + + fw_upload_priv = kzalloc(sizeof(*fw_upload_priv), GFP_KERNEL); + if (!fw_upload_priv) { + ret = -ENOMEM; + goto free_fw_upload; + } + + fw_upload_priv->fw_upload = fw_upload; + fw_upload_priv->ops = ops; + mutex_init(&fw_upload_priv->lock); + fw_upload_priv->module = module; + fw_upload_priv->name = name; + fw_upload_priv->err_code = 0; + fw_upload_priv->progress = FW_UPLOAD_PROG_IDLE; + INIT_WORK(&fw_upload_priv->work, fw_upload_main); + fw_upload->dd_handle = dd_handle; + + fw_sysfs = fw_create_instance(NULL, name, parent, opt_flags); + if (IS_ERR(fw_sysfs)) { + ret = PTR_ERR(fw_sysfs); + goto free_fw_upload_priv; + } + fw_upload->priv = fw_sysfs; + fw_sysfs->fw_upload_priv = fw_upload_priv; + fw_dev = &fw_sysfs->dev; + + ret = alloc_lookup_fw_priv(name, &fw_cache, &fw_priv, NULL, 0, 0, + FW_OPT_NOCACHE); + if (ret != 0) { + if (ret > 0) + ret = -EINVAL; + goto free_fw_sysfs; + } + fw_priv->is_paged_buf = true; + fw_sysfs->fw_priv = fw_priv; + + ret = device_add(fw_dev); + if (ret) { + dev_err(fw_dev, "%s: device_register failed\n", __func__); + put_device(fw_dev); + goto exit_module_put; + } + + return fw_upload; + +free_fw_sysfs: + kfree(fw_sysfs); + +free_fw_upload_priv: + kfree(fw_upload_priv); + +free_fw_upload: + kfree(fw_upload); + +exit_module_put: + module_put(module); + + return ERR_PTR(ret); +} +EXPORT_SYMBOL_GPL(firmware_upload_register); + +/** + * firmware_upload_unregister() - Unregister firmware upload interface + * @fw_upload: pointer to struct fw_upload + **/ +void firmware_upload_unregister(struct fw_upload *fw_upload) +{ + struct fw_sysfs *fw_sysfs = fw_upload->priv; + struct fw_upload_priv *fw_upload_priv = fw_sysfs->fw_upload_priv; + struct module *module = fw_upload_priv->module; + + mutex_lock(&fw_upload_priv->lock); + if (fw_upload_priv->progress == FW_UPLOAD_PROG_IDLE) { + mutex_unlock(&fw_upload_priv->lock); + goto unregister; + } + + fw_upload_priv->ops->cancel(fw_upload); + mutex_unlock(&fw_upload_priv->lock); + + /* Ensure lower-level device-driver is finished */ + flush_work(&fw_upload_priv->work); + +unregister: + device_unregister(&fw_sysfs->dev); + module_put(module); +} +EXPORT_SYMBOL_GPL(firmware_upload_unregister); diff --git a/drivers/base/firmware_loader/sysfs_upload.h b/drivers/base/firmware_loader/sysfs_upload.h new file mode 100644 index 000000000000..31931ff7808a --- /dev/null +++ b/drivers/base/firmware_loader/sysfs_upload.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __SYSFS_UPLOAD_H +#define __SYSFS_UPLOAD_H + +#include <linux/device.h> + +#include "sysfs.h" + +/** + * enum fw_upload_prog - firmware upload progress codes + * @FW_UPLOAD_PROG_IDLE: there is no firmware upload in progress + * @FW_UPLOAD_PROG_RECEIVING: worker thread is receiving firmware data + * @FW_UPLOAD_PROG_PREPARING: target device is preparing for firmware upload + * @FW_UPLOAD_PROG_TRANSFERRING: data is being copied to the device + * @FW_UPLOAD_PROG_PROGRAMMING: device is performing the firmware update + * @FW_UPLOAD_PROG_MAX: Maximum progress code marker + */ +enum fw_upload_prog { + FW_UPLOAD_PROG_IDLE, + FW_UPLOAD_PROG_RECEIVING, + FW_UPLOAD_PROG_PREPARING, + FW_UPLOAD_PROG_TRANSFERRING, + FW_UPLOAD_PROG_PROGRAMMING, + FW_UPLOAD_PROG_MAX +}; + +struct fw_upload_priv { + struct fw_upload *fw_upload; + struct module *module; + const char *name; + const struct fw_upload_ops *ops; + struct mutex lock; /* protect data structure contents */ + struct work_struct work; + const u8 *data; /* pointer to update data */ + u32 remaining_size; /* size remaining to transfer */ + enum fw_upload_prog progress; + enum fw_upload_prog err_progress; /* progress at time of failure */ + enum fw_upload_err err_code; /* security manager error code */ +}; + +#endif /* __SYSFS_UPLOAD_H */ diff --git a/drivers/base/hypervisor.c b/drivers/base/hypervisor.c index 4f8b741f4615..1ce59b4b53ce 100644 --- a/drivers/base/hypervisor.c +++ b/drivers/base/hypervisor.c @@ -1,11 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0 /* * hypervisor.c - /sys/hypervisor subsystem. * * Copyright (C) IBM Corp. 2006 * Copyright (C) 2007 Greg Kroah-Hartman <gregkh@suse.de> * Copyright (C) 2007 Novell Inc. - * - * This file is released under the GPLv2 */ #include <linux/kobject.h> diff --git a/drivers/base/init.c b/drivers/base/init.c index c16f0b808a17..9d2b06d65dfc 100644 --- a/drivers/base/init.c +++ b/drivers/base/init.c @@ -1,13 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Copyright (c) 2002-3 Patrick Mochel * Copyright (c) 2002-3 Open Source Development Labs - * - * This file is released under the GPLv2 */ #include <linux/device.h> #include <linux/init.h> #include <linux/memory.h> +#include <linux/of.h> +#include <linux/backing-dev.h> #include "base.h" @@ -20,6 +21,7 @@ void __init driver_init(void) { /* These are the core pieces */ + bdi_init(&noop_backing_dev_info); devtmpfs_init(); devices_init(); buses_init(); @@ -30,7 +32,12 @@ void __init driver_init(void) /* These are also core pieces, but must come after the * core core pieces. */ + faux_bus_init(); + of_core_init(); platform_bus_init(); - cpu_dev_init(); + auxiliary_bus_init(); memory_dev_init(); + node_dev_init(); + cpu_dev_init(); + container_dev_init(); } diff --git a/drivers/base/isa.c b/drivers/base/isa.c index 91dba65d7264..bfd9215c9070 100644 --- a/drivers/base/isa.c +++ b/drivers/base/isa.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * ISA bus. */ @@ -22,7 +23,7 @@ struct isa_dev { #define to_isa_dev(x) container_of((x), struct isa_dev, dev) -static int isa_bus_match(struct device *dev, struct device_driver *driver) +static int isa_bus_match(struct device *dev, const struct device_driver *driver) { struct isa_driver *isa_driver = to_isa_driver(driver); @@ -39,27 +40,25 @@ static int isa_bus_probe(struct device *dev) { struct isa_driver *isa_driver = dev->platform_data; - if (isa_driver->probe) + if (isa_driver && isa_driver->probe) return isa_driver->probe(dev, to_isa_dev(dev)->id); return 0; } -static int isa_bus_remove(struct device *dev) +static void isa_bus_remove(struct device *dev) { struct isa_driver *isa_driver = dev->platform_data; - if (isa_driver->remove) - return isa_driver->remove(dev, to_isa_dev(dev)->id); - - return 0; + if (isa_driver && isa_driver->remove) + isa_driver->remove(dev, to_isa_dev(dev)->id); } static void isa_bus_shutdown(struct device *dev) { struct isa_driver *isa_driver = dev->platform_data; - if (isa_driver->shutdown) + if (isa_driver && isa_driver->shutdown) isa_driver->shutdown(dev, to_isa_dev(dev)->id); } @@ -67,7 +66,7 @@ static int isa_bus_suspend(struct device *dev, pm_message_t state) { struct isa_driver *isa_driver = dev->platform_data; - if (isa_driver->suspend) + if (isa_driver && isa_driver->suspend) return isa_driver->suspend(dev, to_isa_dev(dev)->id, state); return 0; @@ -77,13 +76,13 @@ static int isa_bus_resume(struct device *dev) { struct isa_driver *isa_driver = dev->platform_data; - if (isa_driver->resume) + if (isa_driver && isa_driver->resume) return isa_driver->resume(dev, to_isa_dev(dev)->id); return 0; } -static struct bus_type isa_bus_type = { +static const struct bus_type isa_bus_type = { .name = "isa", .match = isa_bus_match, .probe = isa_bus_probe, @@ -150,11 +149,8 @@ int isa_register_driver(struct isa_driver *isa_driver, unsigned int ndev) break; } - if (isa_dev->dev.platform_data) { - isa_dev->next = isa_driver->devices; - isa_driver->devices = &isa_dev->dev; - } else - device_unregister(&isa_dev->dev); + isa_dev->next = isa_driver->devices; + isa_driver->devices = &isa_dev->dev; } if (!error && !isa_driver->devices) @@ -180,4 +176,4 @@ static int __init isa_bus_init(void) return error; } -device_initcall(isa_bus_init); +postcore_initcall(isa_bus_init); diff --git a/drivers/base/map.c b/drivers/base/map.c index e87017f36853..83aeb09ca161 100644 --- a/drivers/base/map.c +++ b/drivers/base/map.c @@ -1,8 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 /* * linux/drivers/base/map.c * * (C) Copyright Al Viro 2002,2003 - * Released under GPL v2. * * NOTE: data structure needs to be changed. It works, but for large dev_t * it will be too slow. It is isolated, though, so these changes will be @@ -33,16 +33,15 @@ int kobj_map(struct kobj_map *domain, dev_t dev, unsigned long range, struct module *module, kobj_probe_t *probe, int (*lock)(dev_t, void *), void *data) { - unsigned n = MAJOR(dev + range - 1) - MAJOR(dev) + 1; - unsigned index = MAJOR(dev); - unsigned i; + unsigned int n = MAJOR(dev + range - 1) - MAJOR(dev) + 1; + unsigned int index = MAJOR(dev); + unsigned int i; struct probe *p; if (n > 255) n = 255; - p = kmalloc(sizeof(struct probe) * n, GFP_KERNEL); - + p = kmalloc_array(n, sizeof(struct probe), GFP_KERNEL); if (p == NULL) return -ENOMEM; @@ -68,9 +67,9 @@ int kobj_map(struct kobj_map *domain, dev_t dev, unsigned long range, void kobj_unmap(struct kobj_map *domain, dev_t dev, unsigned long range) { - unsigned n = MAJOR(dev + range - 1) - MAJOR(dev) + 1; - unsigned index = MAJOR(dev); - unsigned i; + unsigned int n = MAJOR(dev + range - 1) - MAJOR(dev) + 1; + unsigned int index = MAJOR(dev); + unsigned int i; struct probe *found = NULL; if (n > 255) diff --git a/drivers/base/memory.c b/drivers/base/memory.c index 2b7813ec6d02..751f248ca4a8 100644 --- a/drivers/base/memory.c +++ b/drivers/base/memory.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Memory subsystem support * @@ -16,215 +17,305 @@ #include <linux/capability.h> #include <linux/device.h> #include <linux/memory.h> -#include <linux/kobject.h> #include <linux/memory_hotplug.h> #include <linux/mm.h> -#include <linux/mutex.h> #include <linux/stat.h> #include <linux/slab.h> +#include <linux/xarray.h> +#include <linux/export.h> #include <linux/atomic.h> -#include <asm/uaccess.h> - -static DEFINE_MUTEX(mem_sysfs_mutex); +#include <linux/uaccess.h> #define MEMORY_CLASS_NAME "memory" -static int sections_per_block; +static const char *const online_type_to_str[] = { + [MMOP_OFFLINE] = "offline", + [MMOP_ONLINE] = "online", + [MMOP_ONLINE_KERNEL] = "online_kernel", + [MMOP_ONLINE_MOVABLE] = "online_movable", +}; -static inline int base_memory_block_id(int section_nr) +int mhp_online_type_from_str(const char *str) { - return section_nr / sections_per_block; + int i; + + for (i = 0; i < ARRAY_SIZE(online_type_to_str); i++) { + if (sysfs_streq(str, online_type_to_str[i])) + return i; + } + return -EINVAL; } +#define to_memory_block(dev) container_of(dev, struct memory_block, dev) + +int sections_per_block; +EXPORT_SYMBOL(sections_per_block); + static int memory_subsys_online(struct device *dev); static int memory_subsys_offline(struct device *dev); -static struct bus_type memory_subsys = { +static const struct bus_type memory_subsys = { .name = MEMORY_CLASS_NAME, .dev_name = MEMORY_CLASS_NAME, .online = memory_subsys_online, .offline = memory_subsys_offline, }; +/* + * Memory blocks are cached in a local radix tree to avoid + * a costly linear search for the corresponding device on + * the subsystem bus. + */ +static DEFINE_XARRAY(memory_blocks); + +/* + * Memory groups, indexed by memory group id (mgid). + */ +static DEFINE_XARRAY_FLAGS(memory_groups, XA_FLAGS_ALLOC); +#define MEMORY_GROUP_MARK_DYNAMIC XA_MARK_1 + static BLOCKING_NOTIFIER_HEAD(memory_chain); int register_memory_notifier(struct notifier_block *nb) { - return blocking_notifier_chain_register(&memory_chain, nb); + return blocking_notifier_chain_register(&memory_chain, nb); } EXPORT_SYMBOL(register_memory_notifier); void unregister_memory_notifier(struct notifier_block *nb) { - blocking_notifier_chain_unregister(&memory_chain, nb); + blocking_notifier_chain_unregister(&memory_chain, nb); } EXPORT_SYMBOL(unregister_memory_notifier); -static ATOMIC_NOTIFIER_HEAD(memory_isolate_chain); - -int register_memory_isolate_notifier(struct notifier_block *nb) -{ - return atomic_notifier_chain_register(&memory_isolate_chain, nb); -} -EXPORT_SYMBOL(register_memory_isolate_notifier); - -void unregister_memory_isolate_notifier(struct notifier_block *nb) -{ - atomic_notifier_chain_unregister(&memory_isolate_chain, nb); -} -EXPORT_SYMBOL(unregister_memory_isolate_notifier); - static void memory_block_release(struct device *dev) { - struct memory_block *mem = container_of(dev, struct memory_block, dev); - + struct memory_block *mem = to_memory_block(dev); + /* Verify that the altmap is freed */ + WARN_ON(mem->altmap); kfree(mem); } -unsigned long __weak memory_block_size_bytes(void) -{ - return MIN_MEMORY_BLOCK_SIZE; -} -static unsigned long get_memory_block_size(void) +/* Max block size to be set by memory_block_advise_max_size */ +static unsigned long memory_block_advised_size; +static bool memory_block_advised_size_queried; + +/** + * memory_block_advise_max_size() - advise memory hotplug on the max suggested + * block size, usually for alignment. + * @size: suggestion for maximum block size. must be aligned on power of 2. + * + * Early boot software (pre-allocator init) may advise archs on the max block + * size. This value can only decrease after initialization, as the intent is + * to identify the largest supported alignment for all sources. + * + * Use of this value is arch-defined, as is min/max block size. + * + * Return: 0 on success + * -EINVAL if size is 0 or not pow2 aligned + * -EBUSY if value has already been probed + */ +int __init memory_block_advise_max_size(unsigned long size) { - unsigned long block_sz; + if (!size || !is_power_of_2(size)) + return -EINVAL; - block_sz = memory_block_size_bytes(); + if (memory_block_advised_size_queried) + return -EBUSY; - /* Validate blk_sz is a power of 2 and not less than section size */ - if ((block_sz & (block_sz - 1)) || (block_sz < MIN_MEMORY_BLOCK_SIZE)) { - WARN_ON(1); - block_sz = MIN_MEMORY_BLOCK_SIZE; - } + if (memory_block_advised_size) + memory_block_advised_size = min(memory_block_advised_size, size); + else + memory_block_advised_size = size; - return block_sz; + return 0; } -/* - * use this as the physical section index that this memsection - * uses. +/** + * memory_block_advised_max_size() - query advised max hotplug block size. + * + * After the first call, the value can never change. Callers looking for the + * actual block size should use memory_block_size_bytes. This interface is + * intended for use by arch-init when initializing the hotplug block size. + * + * Return: advised size in bytes, or 0 if never set. */ - -static ssize_t show_mem_start_phys_index(struct device *dev, - struct device_attribute *attr, char *buf) +unsigned long memory_block_advised_max_size(void) { - struct memory_block *mem = - container_of(dev, struct memory_block, dev); - unsigned long phys_index; + memory_block_advised_size_queried = true; + return memory_block_advised_size; +} - phys_index = mem->start_section_nr / sections_per_block; - return sprintf(buf, "%08lx\n", phys_index); +unsigned long __weak memory_block_size_bytes(void) +{ + return MIN_MEMORY_BLOCK_SIZE; } +EXPORT_SYMBOL_GPL(memory_block_size_bytes); -static ssize_t show_mem_end_phys_index(struct device *dev, - struct device_attribute *attr, char *buf) +/* Show the memory block ID, relative to the memory block size */ +static ssize_t phys_index_show(struct device *dev, + struct device_attribute *attr, char *buf) { - struct memory_block *mem = - container_of(dev, struct memory_block, dev); - unsigned long phys_index; + struct memory_block *mem = to_memory_block(dev); - phys_index = mem->end_section_nr / sections_per_block; - return sprintf(buf, "%08lx\n", phys_index); + return sysfs_emit(buf, "%08lx\n", memory_block_id(mem->start_section_nr)); } /* - * Show whether the section of memory is likely to be hot-removable + * Legacy interface that we cannot remove. Always indicate "removable" + * with CONFIG_MEMORY_HOTREMOVE - bad heuristic. */ -static ssize_t show_mem_removable(struct device *dev, - struct device_attribute *attr, char *buf) +static ssize_t removable_show(struct device *dev, struct device_attribute *attr, + char *buf) { - unsigned long i, pfn; - int ret = 1; - struct memory_block *mem = - container_of(dev, struct memory_block, dev); - - for (i = 0; i < sections_per_block; i++) { - pfn = section_nr_to_pfn(mem->start_section_nr + i); - ret &= is_mem_section_removable(pfn, PAGES_PER_SECTION); - } - - return sprintf(buf, "%d\n", ret); + return sysfs_emit(buf, "%d\n", (int)IS_ENABLED(CONFIG_MEMORY_HOTREMOVE)); } /* * online, offline, going offline, etc. */ -static ssize_t show_mem_state(struct device *dev, - struct device_attribute *attr, char *buf) +static ssize_t state_show(struct device *dev, struct device_attribute *attr, + char *buf) { - struct memory_block *mem = - container_of(dev, struct memory_block, dev); - ssize_t len = 0; + struct memory_block *mem = to_memory_block(dev); + const char *output; /* * We can probably put these states in a nice little array * so that they're not open-coded */ switch (mem->state) { - case MEM_ONLINE: - len = sprintf(buf, "online\n"); - break; - case MEM_OFFLINE: - len = sprintf(buf, "offline\n"); - break; - case MEM_GOING_OFFLINE: - len = sprintf(buf, "going-offline\n"); - break; - default: - len = sprintf(buf, "ERROR-UNKNOWN-%ld\n", - mem->state); - WARN_ON(1); - break; + case MEM_ONLINE: + output = "online"; + break; + case MEM_OFFLINE: + output = "offline"; + break; + case MEM_GOING_OFFLINE: + output = "going-offline"; + break; + default: + WARN_ON(1); + return sysfs_emit(buf, "ERROR-UNKNOWN-%d\n", mem->state); } - return len; + return sysfs_emit(buf, "%s\n", output); } -int memory_notify(unsigned long val, void *v) +int memory_notify(enum memory_block_state state, void *v) { - return blocking_notifier_call_chain(&memory_chain, val, v); + return blocking_notifier_call_chain(&memory_chain, state, v); } -int memory_isolate_notify(unsigned long val, void *v) +#if defined(CONFIG_MEMORY_FAILURE) && defined(CONFIG_MEMORY_HOTPLUG) +static unsigned long memblk_nr_poison(struct memory_block *mem); +#else +static inline unsigned long memblk_nr_poison(struct memory_block *mem) { - return atomic_notifier_call_chain(&memory_isolate_chain, val, v); + return 0; } +#endif /* - * The probe routines leave the pages reserved, just as the bootmem code does. - * Make sure they're still that way. + * Must acquire mem_hotplug_lock in write mode. */ -static bool pages_correctly_reserved(unsigned long start_pfn) +static int memory_block_online(struct memory_block *mem) { - int i, j; - struct page *page; - unsigned long pfn = start_pfn; + unsigned long start_pfn = section_nr_to_pfn(mem->start_section_nr); + unsigned long nr_pages = PAGES_PER_SECTION * sections_per_block; + unsigned long nr_vmemmap_pages = 0; + struct zone *zone; + int ret; + + if (memblk_nr_poison(mem)) + return -EHWPOISON; + + zone = zone_for_pfn_range(mem->online_type, mem->nid, mem->group, + start_pfn, nr_pages); /* - * memmap between sections is not contiguous except with - * SPARSEMEM_VMEMMAP. We lookup the page once per section - * and assume memmap is contiguous within each section + * Although vmemmap pages have a different lifecycle than the pages + * they describe (they remain until the memory is unplugged), doing + * their initialization and accounting at memory onlining/offlining + * stage helps to keep accounting easier to follow - e.g vmemmaps + * belong to the same zone as the memory they backed. */ - for (i = 0; i < sections_per_block; i++, pfn += PAGES_PER_SECTION) { - if (WARN_ON_ONCE(!pfn_valid(pfn))) - return false; - page = pfn_to_page(pfn); + if (mem->altmap) + nr_vmemmap_pages = mem->altmap->free; - for (j = 0; j < PAGES_PER_SECTION; j++) { - if (PageReserved(page + j)) - continue; + mem_hotplug_begin(); + if (nr_vmemmap_pages) { + ret = mhp_init_memmap_on_memory(start_pfn, nr_vmemmap_pages, zone); + if (ret) + goto out; + } - printk(KERN_WARNING "section number %ld page number %d " - "not reserved, was it already online?\n", - pfn_to_section_nr(pfn), j); + ret = online_pages(start_pfn + nr_vmemmap_pages, + nr_pages - nr_vmemmap_pages, zone, mem->group); + if (ret) { + if (nr_vmemmap_pages) + mhp_deinit_memmap_on_memory(start_pfn, nr_vmemmap_pages); + goto out; + } - return false; - } + /* + * Account once onlining succeeded. If the zone was unpopulated, it is + * now already properly populated. + */ + if (nr_vmemmap_pages) + adjust_present_page_count(pfn_to_page(start_pfn), mem->group, + nr_vmemmap_pages); + + mem->zone = zone; +out: + mem_hotplug_done(); + return ret; +} + +/* + * Must acquire mem_hotplug_lock in write mode. + */ +static int memory_block_offline(struct memory_block *mem) +{ + unsigned long start_pfn = section_nr_to_pfn(mem->start_section_nr); + unsigned long nr_pages = PAGES_PER_SECTION * sections_per_block; + unsigned long nr_vmemmap_pages = 0; + int ret; + + if (!mem->zone) + return -EINVAL; + + /* + * Unaccount before offlining, such that unpopulated zone and kthreads + * can properly be torn down in offline_pages(). + */ + if (mem->altmap) + nr_vmemmap_pages = mem->altmap->free; + + mem_hotplug_begin(); + if (nr_vmemmap_pages) + adjust_present_page_count(pfn_to_page(start_pfn), mem->group, + -nr_vmemmap_pages); + + ret = offline_pages(start_pfn + nr_vmemmap_pages, + nr_pages - nr_vmemmap_pages, mem->zone, mem->group); + if (ret) { + /* offline_pages() failed. Account back. */ + if (nr_vmemmap_pages) + adjust_present_page_count(pfn_to_page(start_pfn), + mem->group, nr_vmemmap_pages); + goto out; } - return true; + if (nr_vmemmap_pages) + mhp_deinit_memmap_on_memory(start_pfn, nr_vmemmap_pages); + + mem->zone = NULL; +out: + mem_hotplug_done(); + return ret; } /* @@ -232,38 +323,28 @@ static bool pages_correctly_reserved(unsigned long start_pfn) * OK to have direct references to sparsemem variables in here. */ static int -memory_block_action(unsigned long phys_index, unsigned long action, int online_type) +memory_block_action(struct memory_block *mem, unsigned long action) { - unsigned long start_pfn; - unsigned long nr_pages = PAGES_PER_SECTION * sections_per_block; - struct page *first_page; int ret; - first_page = pfn_to_page(phys_index << PFN_SECTION_SHIFT); - start_pfn = page_to_pfn(first_page); - switch (action) { - case MEM_ONLINE: - if (!pages_correctly_reserved(start_pfn)) - return -EBUSY; - - ret = online_pages(start_pfn, nr_pages, online_type); - break; - case MEM_OFFLINE: - ret = offline_pages(start_pfn, nr_pages); - break; - default: - WARN(1, KERN_WARNING "%s(%ld, %ld) unknown action: " - "%ld\n", __func__, phys_index, action, action); - ret = -EINVAL; + case MEM_ONLINE: + ret = memory_block_online(mem); + break; + case MEM_OFFLINE: + ret = memory_block_offline(mem); + break; + default: + WARN(1, KERN_WARNING "%s(%ld, %ld) unknown action: " + "%ld\n", __func__, mem->start_section_nr, action, action); + ret = -EINVAL; } return ret; } -static int __memory_block_change_state(struct memory_block *mem, - unsigned long to_state, unsigned long from_state_req, - int online_type) +static int memory_block_change_state(struct memory_block *mem, + unsigned long to_state, unsigned long from_state_req) { int ret = 0; @@ -273,147 +354,204 @@ static int __memory_block_change_state(struct memory_block *mem, if (to_state == MEM_OFFLINE) mem->state = MEM_GOING_OFFLINE; - ret = memory_block_action(mem->start_section_nr, to_state, online_type); + ret = memory_block_action(mem, to_state); mem->state = ret ? from_state_req : to_state; + return ret; } +/* The device lock serializes operations on memory_subsys_[online|offline] */ static int memory_subsys_online(struct device *dev) { - struct memory_block *mem = container_of(dev, struct memory_block, dev); + struct memory_block *mem = to_memory_block(dev); int ret; - mutex_lock(&mem->state_mutex); + if (mem->state == MEM_ONLINE) + return 0; + + /* + * When called via device_online() without configuring the online_type, + * we want to default to MMOP_ONLINE. + */ + if (mem->online_type == MMOP_OFFLINE) + mem->online_type = MMOP_ONLINE; - ret = mem->state == MEM_ONLINE ? 0 : - __memory_block_change_state(mem, MEM_ONLINE, MEM_OFFLINE, - ONLINE_KEEP); + ret = memory_block_change_state(mem, MEM_ONLINE, MEM_OFFLINE); + mem->online_type = MMOP_OFFLINE; - mutex_unlock(&mem->state_mutex); return ret; } static int memory_subsys_offline(struct device *dev) { - struct memory_block *mem = container_of(dev, struct memory_block, dev); + struct memory_block *mem = to_memory_block(dev); + + if (mem->state == MEM_OFFLINE) + return 0; + + return memory_block_change_state(mem, MEM_OFFLINE, MEM_ONLINE); +} + +static ssize_t state_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + const int online_type = mhp_online_type_from_str(buf); + struct memory_block *mem = to_memory_block(dev); int ret; - mutex_lock(&mem->state_mutex); + if (online_type < 0) + return -EINVAL; + + ret = lock_device_hotplug_sysfs(); + if (ret) + return ret; - ret = mem->state == MEM_OFFLINE ? 0 : - __memory_block_change_state(mem, MEM_OFFLINE, MEM_ONLINE, -1); + switch (online_type) { + case MMOP_ONLINE_KERNEL: + case MMOP_ONLINE_MOVABLE: + case MMOP_ONLINE: + /* mem->online_type is protected by device_hotplug_lock */ + mem->online_type = online_type; + ret = device_online(&mem->dev); + break; + case MMOP_OFFLINE: + ret = device_offline(&mem->dev); + break; + default: + ret = -EINVAL; /* should never happen */ + } - mutex_unlock(&mem->state_mutex); - return ret; + unlock_device_hotplug(); + + if (ret < 0) + return ret; + if (ret) + return -EINVAL; + + return count; } -static int __memory_block_change_state_uevent(struct memory_block *mem, - unsigned long to_state, unsigned long from_state_req, - int online_type) +/* + * Legacy interface that we cannot remove: s390x exposes the storage increment + * covered by a memory block, allowing for identifying which memory blocks + * comprise a storage increment. Since a memory block spans complete + * storage increments nowadays, this interface is basically unused. Other + * archs never exposed != 0. + */ +static ssize_t phys_device_show(struct device *dev, + struct device_attribute *attr, char *buf) { - int ret = __memory_block_change_state(mem, to_state, from_state_req, - online_type); - if (!ret) { - switch (mem->state) { - case MEM_OFFLINE: - kobject_uevent(&mem->dev.kobj, KOBJ_OFFLINE); - break; - case MEM_ONLINE: - kobject_uevent(&mem->dev.kobj, KOBJ_ONLINE); - break; - default: - break; - } - } - return ret; + struct memory_block *mem = to_memory_block(dev); + unsigned long start_pfn = section_nr_to_pfn(mem->start_section_nr); + + return sysfs_emit(buf, "%d\n", + arch_get_memory_phys_device(start_pfn)); } -static int memory_block_change_state(struct memory_block *mem, - unsigned long to_state, unsigned long from_state_req, - int online_type) +#ifdef CONFIG_MEMORY_HOTREMOVE +static int print_allowed_zone(char *buf, int len, int nid, + struct memory_group *group, + unsigned long start_pfn, unsigned long nr_pages, + int online_type, struct zone *default_zone) { - int ret; + struct zone *zone; - mutex_lock(&mem->state_mutex); - ret = __memory_block_change_state_uevent(mem, to_state, from_state_req, - online_type); - mutex_unlock(&mem->state_mutex); + zone = zone_for_pfn_range(online_type, nid, group, start_pfn, nr_pages); + if (zone == default_zone) + return 0; - return ret; + return sysfs_emit_at(buf, len, " %s", zone->name); } -static ssize_t -store_mem_state(struct device *dev, - struct device_attribute *attr, const char *buf, size_t count) + +static ssize_t valid_zones_show(struct device *dev, + struct device_attribute *attr, char *buf) { - struct memory_block *mem; - bool offline; - int ret = -EINVAL; - - mem = container_of(dev, struct memory_block, dev); - - lock_device_hotplug(); - - if (!strncmp(buf, "online_kernel", min_t(int, count, 13))) { - offline = false; - ret = memory_block_change_state(mem, MEM_ONLINE, - MEM_OFFLINE, ONLINE_KERNEL); - } else if (!strncmp(buf, "online_movable", min_t(int, count, 14))) { - offline = false; - ret = memory_block_change_state(mem, MEM_ONLINE, - MEM_OFFLINE, ONLINE_MOVABLE); - } else if (!strncmp(buf, "online", min_t(int, count, 6))) { - offline = false; - ret = memory_block_change_state(mem, MEM_ONLINE, - MEM_OFFLINE, ONLINE_KEEP); - } else if(!strncmp(buf, "offline", min_t(int, count, 7))) { - offline = true; - ret = memory_block_change_state(mem, MEM_OFFLINE, - MEM_ONLINE, -1); + struct memory_block *mem = to_memory_block(dev); + unsigned long start_pfn = section_nr_to_pfn(mem->start_section_nr); + unsigned long nr_pages = PAGES_PER_SECTION * sections_per_block; + struct memory_group *group = mem->group; + struct zone *default_zone; + int nid = mem->nid; + int len; + + /* + * Check the existing zone. Make sure that we do that only on the + * online nodes otherwise the page_zone is not reliable + */ + if (mem->state == MEM_ONLINE) { + /* + * If !mem->zone, the memory block spans multiple zones and + * cannot get offlined. + */ + return sysfs_emit(buf, "%s\n", + mem->zone ? mem->zone->name : "none"); } - if (!ret) - dev->offline = offline; - unlock_device_hotplug(); + default_zone = zone_for_pfn_range(MMOP_ONLINE, nid, group, + start_pfn, nr_pages); - if (ret) - return ret; - return count; + len = sysfs_emit(buf, "%s", default_zone->name); + len += print_allowed_zone(buf, len, nid, group, start_pfn, nr_pages, + MMOP_ONLINE_KERNEL, default_zone); + len += print_allowed_zone(buf, len, nid, group, start_pfn, nr_pages, + MMOP_ONLINE_MOVABLE, default_zone); + len += sysfs_emit_at(buf, len, "\n"); + return len; } +static DEVICE_ATTR_RO(valid_zones); +#endif + +static DEVICE_ATTR_RO(phys_index); +static DEVICE_ATTR_RW(state); +static DEVICE_ATTR_RO(phys_device); +static DEVICE_ATTR_RO(removable); /* - * phys_device is a bad name for this. What I really want - * is a way to differentiate between memory ranges that - * are part of physical devices that constitute - * a complete removable unit or fru. - * i.e. do these ranges belong to the same physical device, - * s.t. if I offline all of these sections I can then - * remove the physical device? + * Show the memory block size (shared by all memory blocks). */ -static ssize_t show_phys_device(struct device *dev, - struct device_attribute *attr, char *buf) +static ssize_t block_size_bytes_show(struct device *dev, + struct device_attribute *attr, char *buf) { - struct memory_block *mem = - container_of(dev, struct memory_block, dev); - return sprintf(buf, "%d\n", mem->phys_device); + return sysfs_emit(buf, "%lx\n", memory_block_size_bytes()); } -static DEVICE_ATTR(phys_index, 0444, show_mem_start_phys_index, NULL); -static DEVICE_ATTR(end_phys_index, 0444, show_mem_end_phys_index, NULL); -static DEVICE_ATTR(state, 0644, show_mem_state, store_mem_state); -static DEVICE_ATTR(phys_device, 0444, show_phys_device, NULL); -static DEVICE_ATTR(removable, 0444, show_mem_removable, NULL); +static DEVICE_ATTR_RO(block_size_bytes); /* - * Block size attribute stuff + * Memory auto online policy. */ -static ssize_t -print_block_size(struct device *dev, struct device_attribute *attr, - char *buf) + +static ssize_t auto_online_blocks_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%s\n", + online_type_to_str[mhp_get_default_online_type()]); +} + +static ssize_t auto_online_blocks_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) { - return sprintf(buf, "%lx\n", get_memory_block_size()); + const int online_type = mhp_online_type_from_str(buf); + + if (online_type < 0) + return -EINVAL; + + mhp_set_default_online_type(online_type); + return count; } -static DEVICE_ATTR(block_size_bytes, 0444, print_block_size, NULL); +static DEVICE_ATTR_RW(auto_online_blocks); + +#ifdef CONFIG_CRASH_HOTPLUG +#include <linux/kexec.h> +static ssize_t crash_hotplug_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%d\n", crash_check_hotplug_support()); +} +static DEVICE_ATTR_RO(crash_hotplug); +#endif /* * Some architectures will have custom drivers to do this, and @@ -422,36 +560,39 @@ static DEVICE_ATTR(block_size_bytes, 0444, print_block_size, NULL); * and will require this interface. */ #ifdef CONFIG_ARCH_MEMORY_PROBE -static ssize_t -memory_probe_store(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) +static ssize_t probe_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) { u64 phys_addr; - int nid; - int i, ret; + int nid, ret; unsigned long pages_per_block = PAGES_PER_SECTION * sections_per_block; - phys_addr = simple_strtoull(buf, NULL, 0); + ret = kstrtoull(buf, 0, &phys_addr); + if (ret) + return ret; if (phys_addr & ((pages_per_block << PAGE_SHIFT) - 1)) return -EINVAL; - for (i = 0; i < sections_per_block; i++) { - nid = memory_add_physaddr_to_nid(phys_addr); - ret = add_memory(nid, phys_addr, - PAGES_PER_SECTION << PAGE_SHIFT); - if (ret) - goto out; + ret = lock_device_hotplug_sysfs(); + if (ret) + return ret; - phys_addr += MIN_MEMORY_BLOCK_SIZE; - } + nid = memory_add_physaddr_to_nid(phys_addr); + ret = __add_memory(nid, phys_addr, + MIN_MEMORY_BLOCK_SIZE * sections_per_block, + MHP_NONE); + + if (ret) + goto out; ret = count; out: + unlock_device_hotplug(); return ret; } -static DEVICE_ATTR(probe, S_IWUSR, NULL, memory_probe_store); +static DEVICE_ATTR_WO(probe); #endif #ifdef CONFIG_MEMORY_FAILURE @@ -460,97 +601,86 @@ static DEVICE_ATTR(probe, S_IWUSR, NULL, memory_probe_store); */ /* Soft offline a page */ -static ssize_t -store_soft_offline_page(struct device *dev, - struct device_attribute *attr, - const char *buf, size_t count) +static ssize_t soft_offline_page_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) { int ret; u64 pfn; if (!capable(CAP_SYS_ADMIN)) return -EPERM; - if (strict_strtoull(buf, 0, &pfn) < 0) + if (kstrtoull(buf, 0, &pfn) < 0) return -EINVAL; pfn >>= PAGE_SHIFT; - if (!pfn_valid(pfn)) - return -ENXIO; - ret = soft_offline_page(pfn_to_page(pfn), 0); + ret = soft_offline_page(pfn, 0); return ret == 0 ? count : ret; } /* Forcibly offline a page, including killing processes. */ -static ssize_t -store_hard_offline_page(struct device *dev, - struct device_attribute *attr, - const char *buf, size_t count) +static ssize_t hard_offline_page_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) { int ret; u64 pfn; if (!capable(CAP_SYS_ADMIN)) return -EPERM; - if (strict_strtoull(buf, 0, &pfn) < 0) + if (kstrtoull(buf, 0, &pfn) < 0) return -EINVAL; pfn >>= PAGE_SHIFT; - ret = memory_failure(pfn, 0, 0); + ret = memory_failure(pfn, MF_SW_SIMULATED); + if (ret == -EOPNOTSUPP) + ret = 0; return ret ? ret : count; } -static DEVICE_ATTR(soft_offline_page, S_IWUSR, NULL, store_soft_offline_page); -static DEVICE_ATTR(hard_offline_page, S_IWUSR, NULL, store_hard_offline_page); +static DEVICE_ATTR_WO(soft_offline_page); +static DEVICE_ATTR_WO(hard_offline_page); #endif -/* - * Note that phys_device is optional. It is here to allow for - * differentiation between which *physical* devices each - * section belongs to... - */ +/* See phys_device_show(). */ int __weak arch_get_memory_phys_device(unsigned long start_pfn) { return 0; } /* - * A reference for the returned object is held and the reference for the - * hinted object is released. + * A reference for the returned memory block device is acquired. + * + * Called under device_hotplug_lock. */ -struct memory_block *find_memory_block_hinted(struct mem_section *section, - struct memory_block *hint) +struct memory_block *find_memory_block_by_id(unsigned long block_id) { - int block_id = base_memory_block_id(__section_nr(section)); - struct device *hintdev = hint ? &hint->dev : NULL; - struct device *dev; + struct memory_block *mem; - dev = subsys_find_device_by_id(&memory_subsys, block_id, hintdev); - if (hint) - put_device(&hint->dev); - if (!dev) - return NULL; - return container_of(dev, struct memory_block, dev); + mem = xa_load(&memory_blocks, block_id); + if (mem) + get_device(&mem->dev); + return mem; } /* - * For now, we have a linear search to go find the appropriate - * memory_block corresponding to a particular phys_index. If - * this gets to be a real problem, we can always use a radix - * tree or something here. - * - * This could be made generic for all device subsystems. + * Called under device_hotplug_lock. */ -struct memory_block *find_memory_block(struct mem_section *section) +struct memory_block *find_memory_block(unsigned long section_nr) { - return find_memory_block_hinted(section, NULL); + unsigned long block_id = memory_block_id(section_nr); + + return find_memory_block_by_id(block_id); } static struct attribute *memory_memblk_attrs[] = { &dev_attr_phys_index.attr, - &dev_attr_end_phys_index.attr, &dev_attr_state.attr, &dev_attr_phys_device.attr, &dev_attr_removable.attr, +#ifdef CONFIG_MEMORY_HOTREMOVE + &dev_attr_valid_zones.attr, +#endif NULL }; -static struct attribute_group memory_memblk_attr_group = { +static const struct attribute_group memory_memblk_attr_group = { .attrs = memory_memblk_attrs, }; @@ -559,13 +689,9 @@ static const struct attribute_group *memory_memblk_attr_groups[] = { NULL, }; -/* - * register_memory - Setup a sysfs device for a memory block - */ -static -int register_memory(struct memory_block *memory) +static int __add_memory_block(struct memory_block *memory) { - int error; + int ret; memory->dev.bus = &memory_subsys; memory->dev.id = memory->start_section_nr / sections_per_block; @@ -573,132 +699,222 @@ int register_memory(struct memory_block *memory) memory->dev.groups = memory_memblk_attr_groups; memory->dev.offline = memory->state == MEM_OFFLINE; - error = device_register(&memory->dev); - return error; + ret = device_register(&memory->dev); + if (ret) { + put_device(&memory->dev); + return ret; + } + ret = xa_err(xa_store(&memory_blocks, memory->dev.id, memory, + GFP_KERNEL)); + if (ret) + device_unregister(&memory->dev); + + return ret; } -static int init_memory_block(struct memory_block **memory, - struct mem_section *section, unsigned long state) +static struct zone *early_node_zone_for_memory_block(struct memory_block *mem, + int nid) +{ + const unsigned long start_pfn = section_nr_to_pfn(mem->start_section_nr); + const unsigned long nr_pages = PAGES_PER_SECTION * sections_per_block; + struct zone *zone, *matching_zone = NULL; + pg_data_t *pgdat = NODE_DATA(nid); + int i; + + /* + * This logic only works for early memory, when the applicable zones + * already span the memory block. We don't expect overlapping zones on + * a single node for early memory. So if we're told that some PFNs + * of a node fall into this memory block, we can assume that all node + * zones that intersect with the memory block are actually applicable. + * No need to look at the memmap. + */ + for (i = 0; i < MAX_NR_ZONES; i++) { + zone = pgdat->node_zones + i; + if (!populated_zone(zone)) + continue; + if (!zone_intersects(zone, start_pfn, nr_pages)) + continue; + if (!matching_zone) { + matching_zone = zone; + continue; + } + /* Spans multiple zones ... */ + matching_zone = NULL; + break; + } + return matching_zone; +} + +#ifdef CONFIG_NUMA +/** + * memory_block_add_nid_early() - Indicate that early system RAM falling into + * this memory block device (partially) belongs + * to the given node. + * @mem: The memory block device. + * @nid: The node id. + * + * Indicate that early system RAM falling into this memory block (partially) + * belongs to the given node. This will also properly set/adjust mem->zone based + * on the zone ranges of the given node. + * + * Memory hotplug handles this on memory block creation, where we can only have + * a single nid span a memory block. + */ +void memory_block_add_nid_early(struct memory_block *mem, int nid) +{ + if (mem->nid != nid) { + /* + * For early memory we have to determine the zone when setting + * the node id and handle multiple nodes spanning a single + * memory block by indicate via zone == NULL that we're not + * dealing with a single zone. So if we're setting the node id + * the first time, determine if there is a single zone. If we're + * setting the node id a second time to a different node, + * invalidate the single detected zone. + */ + if (mem->nid == NUMA_NO_NODE) + mem->zone = early_node_zone_for_memory_block(mem, nid); + else + mem->zone = NULL; + /* + * If this memory block spans multiple nodes, we only indicate + * the last processed node. If we span multiple nodes (not applicable + * to hotplugged memory), zone == NULL will prohibit memory offlining + * and consequently unplug. + */ + mem->nid = nid; + } +} +#endif + +static int add_memory_block(unsigned long block_id, int nid, unsigned long state, + struct vmem_altmap *altmap, + struct memory_group *group) { struct memory_block *mem; - unsigned long start_pfn; - int scn_nr; int ret = 0; + mem = find_memory_block_by_id(block_id); + if (mem) { + put_device(&mem->dev); + return -EEXIST; + } mem = kzalloc(sizeof(*mem), GFP_KERNEL); if (!mem) return -ENOMEM; - scn_nr = __section_nr(section); - mem->start_section_nr = - base_memory_block_id(scn_nr) * sections_per_block; - mem->end_section_nr = mem->start_section_nr + sections_per_block - 1; + mem->start_section_nr = block_id * sections_per_block; mem->state = state; - mem->section_count++; - mutex_init(&mem->state_mutex); - start_pfn = section_nr_to_pfn(mem->start_section_nr); - mem->phys_device = arch_get_memory_phys_device(start_pfn); + mem->nid = nid; + mem->altmap = altmap; + INIT_LIST_HEAD(&mem->group_next); + +#ifndef CONFIG_NUMA + if (state == MEM_ONLINE) + /* + * MEM_ONLINE at this point implies early memory. With NUMA, + * we'll determine the zone when setting the node id via + * memory_block_add_nid(). Memory hotplug updated the zone + * manually when memory onlining/offlining succeeds. + */ + mem->zone = early_node_zone_for_memory_block(mem, NUMA_NO_NODE); +#endif /* CONFIG_NUMA */ + + ret = __add_memory_block(mem); + if (ret) + return ret; - ret = register_memory(mem); + if (group) { + mem->group = group; + list_add(&mem->group_next, &group->memory_blocks); + } - *memory = mem; - return ret; + return 0; } -static int add_memory_section(int nid, struct mem_section *section, - struct memory_block **mem_p, - unsigned long state, enum mem_add_context context) +static void remove_memory_block(struct memory_block *memory) { - struct memory_block *mem = NULL; - int scn_nr = __section_nr(section); - int ret = 0; - - mutex_lock(&mem_sysfs_mutex); + if (WARN_ON_ONCE(memory->dev.bus != &memory_subsys)) + return; - if (context == BOOT) { - /* same memory block ? */ - if (mem_p && *mem_p) - if (scn_nr >= (*mem_p)->start_section_nr && - scn_nr <= (*mem_p)->end_section_nr) { - mem = *mem_p; - kobject_get(&mem->dev.kobj); - } - } else - mem = find_memory_block(section); - - if (mem) { - mem->section_count++; - kobject_put(&mem->dev.kobj); - } else { - ret = init_memory_block(&mem, section, state); - /* store memory_block pointer for next loop */ - if (!ret && context == BOOT) - if (mem_p) - *mem_p = mem; - } + WARN_ON(xa_erase(&memory_blocks, memory->dev.id) == NULL); - if (!ret) { - if (context == HOTPLUG && - mem->section_count == sections_per_block) - ret = register_mem_sect_under_node(mem, nid); + if (memory->group) { + list_del(&memory->group_next); + memory->group = NULL; } - mutex_unlock(&mem_sysfs_mutex); - return ret; + /* drop the ref. we got via find_memory_block() */ + put_device(&memory->dev); + device_unregister(&memory->dev); } /* - * need an interface for the VM to add new memory regions, - * but without onlining it. + * Create memory block devices for the given memory area. Start and size + * have to be aligned to memory block granularity. Memory block devices + * will be initialized as offline. + * + * Called under device_hotplug_lock. */ -int register_new_memory(int nid, struct mem_section *section) -{ - return add_memory_section(nid, section, NULL, MEM_OFFLINE, HOTPLUG); -} - -#ifdef CONFIG_MEMORY_HOTREMOVE -static void -unregister_memory(struct memory_block *memory) -{ - BUG_ON(memory->dev.bus != &memory_subsys); - - /* drop the ref. we got in remove_memory_block() */ - kobject_put(&memory->dev.kobj); - device_unregister(&memory->dev); -} - -static int remove_memory_block(unsigned long node_id, - struct mem_section *section, int phys_device) +int create_memory_block_devices(unsigned long start, unsigned long size, + int nid, struct vmem_altmap *altmap, + struct memory_group *group) { + const unsigned long start_block_id = pfn_to_block_id(PFN_DOWN(start)); + unsigned long end_block_id = pfn_to_block_id(PFN_DOWN(start + size)); struct memory_block *mem; + unsigned long block_id; + int ret = 0; - mutex_lock(&mem_sysfs_mutex); - mem = find_memory_block(section); - unregister_mem_sect_under_nodes(mem, __section_nr(section)); - - mem->section_count--; - if (mem->section_count == 0) - unregister_memory(mem); - else - kobject_put(&mem->dev.kobj); + if (WARN_ON_ONCE(!IS_ALIGNED(start, memory_block_size_bytes()) || + !IS_ALIGNED(size, memory_block_size_bytes()))) + return -EINVAL; - mutex_unlock(&mem_sysfs_mutex); - return 0; + for (block_id = start_block_id; block_id != end_block_id; block_id++) { + ret = add_memory_block(block_id, nid, MEM_OFFLINE, altmap, group); + if (ret) + break; + } + if (ret) { + end_block_id = block_id; + for (block_id = start_block_id; block_id != end_block_id; + block_id++) { + mem = find_memory_block_by_id(block_id); + if (WARN_ON_ONCE(!mem)) + continue; + remove_memory_block(mem); + } + } + return ret; } -int unregister_memory_section(struct mem_section *section) +/* + * Remove memory block devices for the given memory area. Start and size + * have to be aligned to memory block granularity. Memory block devices + * have to be offline. + * + * Called under device_hotplug_lock. + */ +void remove_memory_block_devices(unsigned long start, unsigned long size) { - if (!present_section(section)) - return -EINVAL; + const unsigned long start_block_id = pfn_to_block_id(PFN_DOWN(start)); + const unsigned long end_block_id = pfn_to_block_id(PFN_DOWN(start + size)); + struct memory_block *mem; + unsigned long block_id; - return remove_memory_block(0, section, 0); -} -#endif /* CONFIG_MEMORY_HOTREMOVE */ + if (WARN_ON_ONCE(!IS_ALIGNED(start, memory_block_size_bytes()) || + !IS_ALIGNED(size, memory_block_size_bytes()))) + return; -/* return true if the memory block is offlined, otherwise, return false */ -bool is_memblock_offlined(struct memory_block *mem) -{ - return mem->state == MEM_OFFLINE; + for (block_id = start_block_id; block_id != end_block_id; block_id++) { + mem = find_memory_block_by_id(block_id); + if (WARN_ON_ONCE(!mem)) + continue; + num_poisoned_pages_sub(-1UL, memblk_nr_poison(mem)); + unregister_memory_block_under_nodes(mem); + remove_memory_block(mem); + } } static struct attribute *memory_root_attrs[] = { @@ -712,10 +928,14 @@ static struct attribute *memory_root_attrs[] = { #endif &dev_attr_block_size_bytes.attr, + &dev_attr_auto_online_blocks.attr, +#ifdef CONFIG_CRASH_HOTPLUG + &dev_attr_crash_hotplug.attr, +#endif NULL }; -static struct attribute_group memory_root_attr_group = { +static const struct attribute_group memory_root_attr_group = { .attrs = memory_root_attrs, }; @@ -725,41 +945,306 @@ static const struct attribute_group *memory_root_attr_groups[] = { }; /* - * Initialize the sysfs support for memory devices... + * Initialize the sysfs support for memory devices. At the time this function + * is called, we cannot have concurrent creation/deletion of memory block + * devices, the device_hotplug_lock is not needed. */ -int __init memory_dev_init(void) +void __init memory_dev_init(void) { - unsigned int i; int ret; - int err; - unsigned long block_sz; - struct memory_block *mem = NULL; + unsigned long block_sz, block_id, nr; + + /* Validate the configured memory block size */ + block_sz = memory_block_size_bytes(); + if (!is_power_of_2(block_sz) || block_sz < MIN_MEMORY_BLOCK_SIZE) + panic("Memory block size not suitable: 0x%lx\n", block_sz); + sections_per_block = block_sz / MIN_MEMORY_BLOCK_SIZE; ret = subsys_system_register(&memory_subsys, memory_root_attr_groups); if (ret) - goto out; - - block_sz = get_memory_block_size(); - sections_per_block = block_sz / MIN_MEMORY_BLOCK_SIZE; + panic("%s() failed to register subsystem: %d\n", __func__, ret); /* - * Create entries for memory sections that were found - * during boot and have been initialized + * Create entries for memory sections that were found during boot + * and have been initialized. Use @block_id to track the last + * handled block and initialize it to an invalid value (ULONG_MAX) + * to bypass the block ID matching check for the first present + * block so that it can be covered. */ - for (i = 0; i < NR_MEM_SECTIONS; i++) { - if (!present_section_nr(i)) + block_id = ULONG_MAX; + for_each_present_section_nr(0, nr) { + if (block_id != ULONG_MAX && memory_block_id(nr) == block_id) continue; - /* don't need to reuse memory_block if only one per block */ - err = add_memory_section(0, __nr_to_section(i), - (sections_per_block == 1) ? NULL : &mem, - MEM_ONLINE, - BOOT); - if (!ret) - ret = err; + + block_id = memory_block_id(nr); + ret = add_memory_block(block_id, NUMA_NO_NODE, MEM_ONLINE, NULL, NULL); + if (ret) { + panic("%s() failed to add memory block: %d\n", + __func__, ret); + } } +} -out: - if (ret) - printk(KERN_ERR "%s() failed: %d\n", __func__, ret); +/** + * walk_memory_blocks - walk through all present memory blocks overlapped + * by the range [start, start + size) + * + * @start: start address of the memory range + * @size: size of the memory range + * @arg: argument passed to func + * @func: callback for each memory section walked + * + * This function walks through all present memory blocks overlapped by the + * range [start, start + size), calling func on each memory block. + * + * In case func() returns an error, walking is aborted and the error is + * returned. + * + * Called under device_hotplug_lock. + */ +int walk_memory_blocks(unsigned long start, unsigned long size, + void *arg, walk_memory_blocks_func_t func) +{ + const unsigned long start_block_id = phys_to_block_id(start); + const unsigned long end_block_id = phys_to_block_id(start + size - 1); + struct memory_block *mem; + unsigned long block_id; + int ret = 0; + + if (!size) + return 0; + + for (block_id = start_block_id; block_id <= end_block_id; block_id++) { + mem = find_memory_block_by_id(block_id); + if (!mem) + continue; + + ret = func(mem, arg); + put_device(&mem->dev); + if (ret) + break; + } + return ret; +} + +struct for_each_memory_block_cb_data { + walk_memory_blocks_func_t func; + void *arg; +}; + +static int for_each_memory_block_cb(struct device *dev, void *data) +{ + struct memory_block *mem = to_memory_block(dev); + struct for_each_memory_block_cb_data *cb_data = data; + + return cb_data->func(mem, cb_data->arg); +} + +/** + * for_each_memory_block - walk through all present memory blocks + * + * @arg: argument passed to func + * @func: callback for each memory block walked + * + * This function walks through all present memory blocks, calling func on + * each memory block. + * + * In case func() returns an error, walking is aborted and the error is + * returned. + */ +int for_each_memory_block(void *arg, walk_memory_blocks_func_t func) +{ + struct for_each_memory_block_cb_data cb_data = { + .func = func, + .arg = arg, + }; + + return bus_for_each_dev(&memory_subsys, NULL, &cb_data, + for_each_memory_block_cb); +} + +/* + * This is an internal helper to unify allocation and initialization of + * memory groups. Note that the passed memory group will be copied to a + * dynamically allocated memory group. After this call, the passed + * memory group should no longer be used. + */ +static int memory_group_register(struct memory_group group) +{ + struct memory_group *new_group; + uint32_t mgid; + int ret; + + if (!node_possible(group.nid)) + return -EINVAL; + + new_group = kzalloc(sizeof(group), GFP_KERNEL); + if (!new_group) + return -ENOMEM; + *new_group = group; + INIT_LIST_HEAD(&new_group->memory_blocks); + + ret = xa_alloc(&memory_groups, &mgid, new_group, xa_limit_31b, + GFP_KERNEL); + if (ret) { + kfree(new_group); + return ret; + } else if (group.is_dynamic) { + xa_set_mark(&memory_groups, mgid, MEMORY_GROUP_MARK_DYNAMIC); + } + return mgid; +} + +/** + * memory_group_register_static() - Register a static memory group. + * @nid: The node id. + * @max_pages: The maximum number of pages we'll have in this static memory + * group. + * + * Register a new static memory group and return the memory group id. + * All memory in the group belongs to a single unit, such as a DIMM. All + * memory belonging to a static memory group is added in one go to be removed + * in one go -- it's static. + * + * Returns an error if out of memory, if the node id is invalid, if no new + * memory groups can be registered, or if max_pages is invalid (0). Otherwise, + * returns the new memory group id. + */ +int memory_group_register_static(int nid, unsigned long max_pages) +{ + struct memory_group group = { + .nid = nid, + .s = { + .max_pages = max_pages, + }, + }; + + if (!max_pages) + return -EINVAL; + return memory_group_register(group); +} +EXPORT_SYMBOL_GPL(memory_group_register_static); + +/** + * memory_group_register_dynamic() - Register a dynamic memory group. + * @nid: The node id. + * @unit_pages: Unit in pages in which is memory added/removed in this dynamic + * memory group. + * + * Register a new dynamic memory group and return the memory group id. + * Memory within a dynamic memory group is added/removed dynamically + * in unit_pages. + * + * Returns an error if out of memory, if the node id is invalid, if no new + * memory groups can be registered, or if unit_pages is invalid (0, not a + * power of two, smaller than a single memory block). Otherwise, returns the + * new memory group id. + */ +int memory_group_register_dynamic(int nid, unsigned long unit_pages) +{ + struct memory_group group = { + .nid = nid, + .is_dynamic = true, + .d = { + .unit_pages = unit_pages, + }, + }; + + if (!unit_pages || !is_power_of_2(unit_pages) || + unit_pages < PHYS_PFN(memory_block_size_bytes())) + return -EINVAL; + return memory_group_register(group); +} +EXPORT_SYMBOL_GPL(memory_group_register_dynamic); + +/** + * memory_group_unregister() - Unregister a memory group. + * @mgid: the memory group id + * + * Unregister a memory group. If any memory block still belongs to this + * memory group, unregistering will fail. + * + * Returns -EINVAL if the memory group id is invalid, returns -EBUSY if some + * memory blocks still belong to this memory group and returns 0 if + * unregistering succeeded. + */ +int memory_group_unregister(int mgid) +{ + struct memory_group *group; + + if (mgid < 0) + return -EINVAL; + + group = xa_load(&memory_groups, mgid); + if (!group) + return -EINVAL; + if (!list_empty(&group->memory_blocks)) + return -EBUSY; + xa_erase(&memory_groups, mgid); + kfree(group); + return 0; +} +EXPORT_SYMBOL_GPL(memory_group_unregister); + +/* + * This is an internal helper only to be used in core memory hotplug code to + * lookup a memory group. We don't care about locking, as we don't expect a + * memory group to get unregistered while adding memory to it -- because + * the group and the memory is managed by the same driver. + */ +struct memory_group *memory_group_find_by_id(int mgid) +{ + return xa_load(&memory_groups, mgid); +} + +/* + * This is an internal helper only to be used in core memory hotplug code to + * walk all dynamic memory groups excluding a given memory group, either + * belonging to a specific node, or belonging to any node. + */ +int walk_dynamic_memory_groups(int nid, walk_memory_groups_func_t func, + struct memory_group *excluded, void *arg) +{ + struct memory_group *group; + unsigned long index; + int ret = 0; + + xa_for_each_marked(&memory_groups, index, group, + MEMORY_GROUP_MARK_DYNAMIC) { + if (group == excluded) + continue; +#ifdef CONFIG_NUMA + if (nid != NUMA_NO_NODE && group->nid != nid) + continue; +#endif /* CONFIG_NUMA */ + ret = func(group, arg); + if (ret) + break; + } return ret; } + +#if defined(CONFIG_MEMORY_FAILURE) && defined(CONFIG_MEMORY_HOTPLUG) +void memblk_nr_poison_inc(unsigned long pfn) +{ + const unsigned long block_id = pfn_to_block_id(pfn); + struct memory_block *mem = find_memory_block_by_id(block_id); + + if (mem) + atomic_long_inc(&mem->nr_hwpoison); +} + +void memblk_nr_poison_sub(unsigned long pfn, long i) +{ + const unsigned long block_id = pfn_to_block_id(pfn); + struct memory_block *mem = find_memory_block_by_id(block_id); + + if (mem) + atomic_long_sub(i, &mem->nr_hwpoison); +} + +static unsigned long memblk_nr_poison(struct memory_block *mem) +{ + return atomic_long_read(&mem->nr_hwpoison); +} +#endif diff --git a/drivers/base/module.c b/drivers/base/module.c index db930d3ee312..218aaa096455 100644 --- a/drivers/base/module.c +++ b/drivers/base/module.c @@ -1,8 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0 /* * module.c - module sysfs fun for drivers - * - * This file is released under the GPLv2 - * */ #include <linux/device.h> #include <linux/module.h> @@ -11,7 +9,7 @@ #include <linux/string.h> #include "base.h" -static char *make_driver_name(struct device_driver *drv) +static char *make_driver_name(const struct device_driver *drv) { char *driver_name; @@ -24,52 +22,75 @@ static char *make_driver_name(struct device_driver *drv) static void module_create_drivers_dir(struct module_kobject *mk) { - if (!mk || mk->drivers_dir) - return; + static DEFINE_MUTEX(drivers_dir_mutex); - mk->drivers_dir = kobject_create_and_add("drivers", &mk->kobj); + mutex_lock(&drivers_dir_mutex); + if (mk && !mk->drivers_dir) + mk->drivers_dir = kobject_create_and_add("drivers", &mk->kobj); + mutex_unlock(&drivers_dir_mutex); } -void module_add_driver(struct module *mod, struct device_driver *drv) +int module_add_driver(struct module *mod, const struct device_driver *drv) { char *driver_name; - int no_warn; struct module_kobject *mk = NULL; + int ret; if (!drv) - return; + return 0; if (mod) mk = &mod->mkobj; else if (drv->mod_name) { - struct kobject *mkobj; - - /* Lookup built-in module entry in /sys/modules */ - mkobj = kset_find_obj(module_kset, drv->mod_name); - if (mkobj) { - mk = container_of(mkobj, struct module_kobject, kobj); + /* Lookup or create built-in module entry in /sys/modules */ + mk = lookup_or_create_module_kobject(drv->mod_name); + if (mk) { /* remember our module structure */ drv->p->mkobj = mk; - /* kset_find_obj took a reference */ - kobject_put(mkobj); + /* lookup_or_create_module_kobject took a reference */ + kobject_put(&mk->kobj); } } if (!mk) - return; + return 0; + + ret = sysfs_create_link(&drv->p->kobj, &mk->kobj, "module"); + if (ret) + return ret; - /* Don't check return codes; these calls are idempotent */ - no_warn = sysfs_create_link(&drv->p->kobj, &mk->kobj, "module"); driver_name = make_driver_name(drv); - if (driver_name) { - module_create_drivers_dir(mk); - no_warn = sysfs_create_link(mk->drivers_dir, &drv->p->kobj, - driver_name); - kfree(driver_name); + if (!driver_name) { + ret = -ENOMEM; + goto out_remove_kobj; + } + + module_create_drivers_dir(mk); + if (!mk->drivers_dir) { + ret = -EINVAL; + goto out_free_driver_name; } + + ret = sysfs_create_link(mk->drivers_dir, &drv->p->kobj, driver_name); + if (ret) + goto out_remove_drivers_dir; + + kfree(driver_name); + + return 0; + +out_remove_drivers_dir: + sysfs_remove_link(mk->drivers_dir, driver_name); + +out_free_driver_name: + kfree(driver_name); + +out_remove_kobj: + sysfs_remove_link(&drv->p->kobj, "module"); + return ret; } -void module_remove_driver(struct device_driver *drv) +void module_remove_driver(const struct device_driver *drv) { struct module_kobject *mk = NULL; char *driver_name; diff --git a/drivers/base/node.c b/drivers/base/node.c index 7616a77ca322..00cf4532f121 100644 --- a/drivers/base/node.c +++ b/drivers/base/node.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Basic Node interface support */ @@ -6,6 +7,7 @@ #include <linux/init.h> #include <linux/mm.h> #include <linux/memory.h> +#include <linux/mempolicy.h> #include <linux/vmstat.h> #include <linux/notifier.h> #include <linux/node.h> @@ -16,183 +18,608 @@ #include <linux/nodemask.h> #include <linux/cpu.h> #include <linux/device.h> +#include <linux/pm_runtime.h> #include <linux/swap.h> #include <linux/slab.h> +#include <linux/memblock.h> -static struct bus_type node_subsys = { +static const struct bus_type node_subsys = { .name = "node", .dev_name = "node", }; +static inline ssize_t cpumap_read(struct file *file, struct kobject *kobj, + const struct bin_attribute *attr, char *buf, + loff_t off, size_t count) +{ + struct device *dev = kobj_to_dev(kobj); + struct node *node_dev = to_node(dev); + cpumask_var_t mask; + ssize_t n; + + if (!alloc_cpumask_var(&mask, GFP_KERNEL)) + return 0; + + cpumask_and(mask, cpumask_of_node(node_dev->dev.id), cpu_online_mask); + n = cpumap_print_bitmask_to_buf(buf, mask, off, count); + free_cpumask_var(mask); -static ssize_t node_read_cpumap(struct device *dev, int type, char *buf) + return n; +} + +static const BIN_ATTR_RO(cpumap, CPUMAP_FILE_MAX_BYTES); + +static inline ssize_t cpulist_read(struct file *file, struct kobject *kobj, + const struct bin_attribute *attr, char *buf, + loff_t off, size_t count) { + struct device *dev = kobj_to_dev(kobj); struct node *node_dev = to_node(dev); - const struct cpumask *mask = cpumask_of_node(node_dev->dev.id); - int len; + cpumask_var_t mask; + ssize_t n; - /* 2008/04/07: buf currently PAGE_SIZE, need 9 chars per 32 bits. */ - BUILD_BUG_ON((NR_CPUS/32 * 9) > (PAGE_SIZE-1)); + if (!alloc_cpumask_var(&mask, GFP_KERNEL)) + return 0; - len = type? - cpulist_scnprintf(buf, PAGE_SIZE-2, mask) : - cpumask_scnprintf(buf, PAGE_SIZE-2, mask); - buf[len++] = '\n'; - buf[len] = '\0'; - return len; + cpumask_and(mask, cpumask_of_node(node_dev->dev.id), cpu_online_mask); + n = cpumap_print_list_to_buf(buf, mask, off, count); + free_cpumask_var(mask); + + return n; } -static inline ssize_t node_read_cpumask(struct device *dev, - struct device_attribute *attr, char *buf) +static const BIN_ATTR_RO(cpulist, CPULIST_FILE_MAX_BYTES); + +/** + * struct node_access_nodes - Access class device to hold user visible + * relationships to other nodes. + * @dev: Device for this memory access class + * @list_node: List element in the node's access list + * @access: The access class rank + * @coord: Heterogeneous memory performance coordinates + */ +struct node_access_nodes { + struct device dev; + struct list_head list_node; + unsigned int access; +#ifdef CONFIG_HMEM_REPORTING + struct access_coordinate coord; +#endif +}; +#define to_access_nodes(dev) container_of(dev, struct node_access_nodes, dev) + +static struct attribute *node_init_access_node_attrs[] = { + NULL, +}; + +static struct attribute *node_targ_access_node_attrs[] = { + NULL, +}; + +static const struct attribute_group initiators = { + .name = "initiators", + .attrs = node_init_access_node_attrs, +}; + +static const struct attribute_group targets = { + .name = "targets", + .attrs = node_targ_access_node_attrs, +}; + +static const struct attribute_group *node_access_node_groups[] = { + &initiators, + &targets, + NULL, +}; + +#ifdef CONFIG_MEMORY_HOTPLUG +static BLOCKING_NOTIFIER_HEAD(node_chain); + +int register_node_notifier(struct notifier_block *nb) { - return node_read_cpumap(dev, 0, buf); + return blocking_notifier_chain_register(&node_chain, nb); } -static inline ssize_t node_read_cpulist(struct device *dev, - struct device_attribute *attr, char *buf) +EXPORT_SYMBOL(register_node_notifier); + +void unregister_node_notifier(struct notifier_block *nb) +{ + blocking_notifier_chain_unregister(&node_chain, nb); +} +EXPORT_SYMBOL(unregister_node_notifier); + +int node_notify(unsigned long val, void *v) +{ + return blocking_notifier_call_chain(&node_chain, val, v); +} +#endif + +static void node_remove_accesses(struct node *node) +{ + struct node_access_nodes *c, *cnext; + + list_for_each_entry_safe(c, cnext, &node->access_list, list_node) { + list_del(&c->list_node); + device_unregister(&c->dev); + } +} + +static void node_access_release(struct device *dev) +{ + kfree(to_access_nodes(dev)); +} + +static struct node_access_nodes *node_init_node_access(struct node *node, + enum access_coordinate_class access) +{ + struct node_access_nodes *access_node; + struct device *dev; + + list_for_each_entry(access_node, &node->access_list, list_node) + if (access_node->access == access) + return access_node; + + access_node = kzalloc(sizeof(*access_node), GFP_KERNEL); + if (!access_node) + return NULL; + + access_node->access = access; + dev = &access_node->dev; + dev->parent = &node->dev; + dev->release = node_access_release; + dev->groups = node_access_node_groups; + if (dev_set_name(dev, "access%u", access)) + goto free; + + if (device_register(dev)) + goto free_name; + + pm_runtime_no_callbacks(dev); + list_add_tail(&access_node->list_node, &node->access_list); + return access_node; +free_name: + kfree_const(dev->kobj.name); +free: + kfree(access_node); + return NULL; +} + +#ifdef CONFIG_HMEM_REPORTING +#define ACCESS_ATTR(property) \ +static ssize_t property##_show(struct device *dev, \ + struct device_attribute *attr, \ + char *buf) \ +{ \ + return sysfs_emit(buf, "%u\n", \ + to_access_nodes(dev)->coord.property); \ +} \ +static DEVICE_ATTR_RO(property) + +ACCESS_ATTR(read_bandwidth); +ACCESS_ATTR(read_latency); +ACCESS_ATTR(write_bandwidth); +ACCESS_ATTR(write_latency); + +static struct attribute *access_attrs[] = { + &dev_attr_read_bandwidth.attr, + &dev_attr_read_latency.attr, + &dev_attr_write_bandwidth.attr, + &dev_attr_write_latency.attr, + NULL, +}; + +/** + * node_set_perf_attrs - Set the performance values for given access class + * @nid: Node identifier to be set + * @coord: Heterogeneous memory performance coordinates + * @access: The access class the for the given attributes + */ +void node_set_perf_attrs(unsigned int nid, struct access_coordinate *coord, + enum access_coordinate_class access) +{ + struct node_access_nodes *c; + struct node *node; + int i; + + if (WARN_ON_ONCE(!node_online(nid))) + return; + + node = node_devices[nid]; + c = node_init_node_access(node, access); + if (!c) + return; + + c->coord = *coord; + for (i = 0; access_attrs[i] != NULL; i++) { + if (sysfs_add_file_to_group(&c->dev.kobj, access_attrs[i], + "initiators")) { + pr_info("failed to add performance attribute to node %d\n", + nid); + break; + } + } + + /* When setting CPU access coordinates, update mempolicy */ + if (access == ACCESS_COORDINATE_CPU) { + if (mempolicy_set_node_perf(nid, coord)) { + pr_info("failed to set mempolicy attrs for node %d\n", + nid); + } + } +} +EXPORT_SYMBOL_GPL(node_set_perf_attrs); + +/** + * node_update_perf_attrs - Update the performance values for given access class + * @nid: Node identifier to be updated + * @coord: Heterogeneous memory performance coordinates + * @access: The access class for the given attributes + */ +void node_update_perf_attrs(unsigned int nid, struct access_coordinate *coord, + enum access_coordinate_class access) +{ + struct node_access_nodes *access_node; + struct node *node; + int i; + + if (WARN_ON_ONCE(!node_online(nid))) + return; + + node = node_devices[nid]; + list_for_each_entry(access_node, &node->access_list, list_node) { + if (access_node->access != access) + continue; + + access_node->coord = *coord; + for (i = 0; access_attrs[i]; i++) { + sysfs_notify(&access_node->dev.kobj, + NULL, access_attrs[i]->name); + } + break; + } + + /* When setting CPU access coordinates, update mempolicy */ + if (access != ACCESS_COORDINATE_CPU) + return; + + if (mempolicy_set_node_perf(nid, coord)) + pr_info("failed to set mempolicy attrs for node %d\n", nid); +} +EXPORT_SYMBOL_GPL(node_update_perf_attrs); + +/** + * struct node_cache_info - Internal tracking for memory node caches + * @dev: Device represeting the cache level + * @node: List element for tracking in the node + * @cache_attrs:Attributes for this cache level + */ +struct node_cache_info { + struct device dev; + struct list_head node; + struct node_cache_attrs cache_attrs; +}; +#define to_cache_info(device) container_of(device, struct node_cache_info, dev) + +#define CACHE_ATTR(name, fmt) \ +static ssize_t name##_show(struct device *dev, \ + struct device_attribute *attr, \ + char *buf) \ +{ \ + return sysfs_emit(buf, fmt "\n", \ + to_cache_info(dev)->cache_attrs.name); \ +} \ +static DEVICE_ATTR_RO(name); + +CACHE_ATTR(size, "%llu") +CACHE_ATTR(line_size, "%u") +CACHE_ATTR(indexing, "%u") +CACHE_ATTR(write_policy, "%u") +CACHE_ATTR(address_mode, "%#x") + +static struct attribute *cache_attrs[] = { + &dev_attr_indexing.attr, + &dev_attr_size.attr, + &dev_attr_line_size.attr, + &dev_attr_write_policy.attr, + &dev_attr_address_mode.attr, + NULL, +}; +ATTRIBUTE_GROUPS(cache); + +static void node_cache_release(struct device *dev) +{ + kfree(dev); +} + +static void node_cacheinfo_release(struct device *dev) { - return node_read_cpumap(dev, 1, buf); + struct node_cache_info *info = to_cache_info(dev); + kfree(info); +} + +static void node_init_cache_dev(struct node *node) +{ + struct device *dev; + + dev = kzalloc(sizeof(*dev), GFP_KERNEL); + if (!dev) + return; + + device_initialize(dev); + dev->parent = &node->dev; + dev->release = node_cache_release; + if (dev_set_name(dev, "memory_side_cache")) + goto put_device; + + if (device_add(dev)) + goto put_device; + + pm_runtime_no_callbacks(dev); + node->cache_dev = dev; + return; +put_device: + put_device(dev); +} + +/** + * node_add_cache() - add cache attribute to a memory node + * @nid: Node identifier that has new cache attributes + * @cache_attrs: Attributes for the cache being added + */ +void node_add_cache(unsigned int nid, struct node_cache_attrs *cache_attrs) +{ + struct node_cache_info *info; + struct device *dev; + struct node *node; + + if (!node_online(nid) || !node_devices[nid]) + return; + + node = node_devices[nid]; + list_for_each_entry(info, &node->cache_attrs, node) { + if (info->cache_attrs.level == cache_attrs->level) { + dev_warn(&node->dev, + "attempt to add duplicate cache level:%d\n", + cache_attrs->level); + return; + } + } + + if (!node->cache_dev) + node_init_cache_dev(node); + if (!node->cache_dev) + return; + + info = kzalloc(sizeof(*info), GFP_KERNEL); + if (!info) + return; + + dev = &info->dev; + device_initialize(dev); + dev->parent = node->cache_dev; + dev->release = node_cacheinfo_release; + dev->groups = cache_groups; + if (dev_set_name(dev, "index%d", cache_attrs->level)) + goto put_device; + + info->cache_attrs = *cache_attrs; + if (device_add(dev)) { + dev_warn(&node->dev, "failed to add cache level:%d\n", + cache_attrs->level); + goto put_device; + } + pm_runtime_no_callbacks(dev); + list_add_tail(&info->node, &node->cache_attrs); + return; +put_device: + put_device(dev); +} + +static void node_remove_caches(struct node *node) +{ + struct node_cache_info *info, *next; + + if (!node->cache_dev) + return; + + list_for_each_entry_safe(info, next, &node->cache_attrs, node) { + list_del(&info->node); + device_unregister(&info->dev); + } + device_unregister(node->cache_dev); } -static DEVICE_ATTR(cpumap, S_IRUGO, node_read_cpumask, NULL); -static DEVICE_ATTR(cpulist, S_IRUGO, node_read_cpulist, NULL); +static void node_init_caches(unsigned int nid) +{ + INIT_LIST_HEAD(&node_devices[nid]->cache_attrs); +} +#else +static void node_init_caches(unsigned int nid) { } +static void node_remove_caches(struct node *node) { } +#endif #define K(x) ((x) << (PAGE_SHIFT - 10)) static ssize_t node_read_meminfo(struct device *dev, struct device_attribute *attr, char *buf) { - int n; + int len = 0; int nid = dev->id; + struct pglist_data *pgdat = NODE_DATA(nid); struct sysinfo i; + unsigned long sreclaimable, sunreclaimable; + unsigned long swapcached = 0; si_meminfo_node(&i, nid); - n = sprintf(buf, - "Node %d MemTotal: %8lu kB\n" - "Node %d MemFree: %8lu kB\n" - "Node %d MemUsed: %8lu kB\n" - "Node %d Active: %8lu kB\n" - "Node %d Inactive: %8lu kB\n" - "Node %d Active(anon): %8lu kB\n" - "Node %d Inactive(anon): %8lu kB\n" - "Node %d Active(file): %8lu kB\n" - "Node %d Inactive(file): %8lu kB\n" - "Node %d Unevictable: %8lu kB\n" - "Node %d Mlocked: %8lu kB\n", - nid, K(i.totalram), - nid, K(i.freeram), - nid, K(i.totalram - i.freeram), - nid, K(node_page_state(nid, NR_ACTIVE_ANON) + - node_page_state(nid, NR_ACTIVE_FILE)), - nid, K(node_page_state(nid, NR_INACTIVE_ANON) + - node_page_state(nid, NR_INACTIVE_FILE)), - nid, K(node_page_state(nid, NR_ACTIVE_ANON)), - nid, K(node_page_state(nid, NR_INACTIVE_ANON)), - nid, K(node_page_state(nid, NR_ACTIVE_FILE)), - nid, K(node_page_state(nid, NR_INACTIVE_FILE)), - nid, K(node_page_state(nid, NR_UNEVICTABLE)), - nid, K(node_page_state(nid, NR_MLOCK))); + sreclaimable = node_page_state_pages(pgdat, NR_SLAB_RECLAIMABLE_B); + sunreclaimable = node_page_state_pages(pgdat, NR_SLAB_UNRECLAIMABLE_B); +#ifdef CONFIG_SWAP + swapcached = node_page_state_pages(pgdat, NR_SWAPCACHE); +#endif + len = sysfs_emit_at(buf, len, + "Node %d MemTotal: %8lu kB\n" + "Node %d MemFree: %8lu kB\n" + "Node %d MemUsed: %8lu kB\n" + "Node %d SwapCached: %8lu kB\n" + "Node %d Active: %8lu kB\n" + "Node %d Inactive: %8lu kB\n" + "Node %d Active(anon): %8lu kB\n" + "Node %d Inactive(anon): %8lu kB\n" + "Node %d Active(file): %8lu kB\n" + "Node %d Inactive(file): %8lu kB\n" + "Node %d Unevictable: %8lu kB\n" + "Node %d Mlocked: %8lu kB\n", + nid, K(i.totalram), + nid, K(i.freeram), + nid, K(i.totalram - i.freeram), + nid, K(swapcached), + nid, K(node_page_state(pgdat, NR_ACTIVE_ANON) + + node_page_state(pgdat, NR_ACTIVE_FILE)), + nid, K(node_page_state(pgdat, NR_INACTIVE_ANON) + + node_page_state(pgdat, NR_INACTIVE_FILE)), + nid, K(node_page_state(pgdat, NR_ACTIVE_ANON)), + nid, K(node_page_state(pgdat, NR_INACTIVE_ANON)), + nid, K(node_page_state(pgdat, NR_ACTIVE_FILE)), + nid, K(node_page_state(pgdat, NR_INACTIVE_FILE)), + nid, K(node_page_state(pgdat, NR_UNEVICTABLE)), + nid, K(sum_zone_node_page_state(nid, NR_MLOCK))); #ifdef CONFIG_HIGHMEM - n += sprintf(buf + n, - "Node %d HighTotal: %8lu kB\n" - "Node %d HighFree: %8lu kB\n" - "Node %d LowTotal: %8lu kB\n" - "Node %d LowFree: %8lu kB\n", - nid, K(i.totalhigh), - nid, K(i.freehigh), - nid, K(i.totalram - i.totalhigh), - nid, K(i.freeram - i.freehigh)); + len += sysfs_emit_at(buf, len, + "Node %d HighTotal: %8lu kB\n" + "Node %d HighFree: %8lu kB\n" + "Node %d LowTotal: %8lu kB\n" + "Node %d LowFree: %8lu kB\n", + nid, K(i.totalhigh), + nid, K(i.freehigh), + nid, K(i.totalram - i.totalhigh), + nid, K(i.freeram - i.freehigh)); #endif - n += sprintf(buf + n, - "Node %d Dirty: %8lu kB\n" - "Node %d Writeback: %8lu kB\n" - "Node %d FilePages: %8lu kB\n" - "Node %d Mapped: %8lu kB\n" - "Node %d AnonPages: %8lu kB\n" - "Node %d Shmem: %8lu kB\n" - "Node %d KernelStack: %8lu kB\n" - "Node %d PageTables: %8lu kB\n" - "Node %d NFS_Unstable: %8lu kB\n" - "Node %d Bounce: %8lu kB\n" - "Node %d WritebackTmp: %8lu kB\n" - "Node %d Slab: %8lu kB\n" - "Node %d SReclaimable: %8lu kB\n" - "Node %d SUnreclaim: %8lu kB\n" -#ifdef CONFIG_TRANSPARENT_HUGEPAGE - "Node %d AnonHugePages: %8lu kB\n" + len += sysfs_emit_at(buf, len, + "Node %d Dirty: %8lu kB\n" + "Node %d Writeback: %8lu kB\n" + "Node %d FilePages: %8lu kB\n" + "Node %d Mapped: %8lu kB\n" + "Node %d AnonPages: %8lu kB\n" + "Node %d Shmem: %8lu kB\n" + "Node %d KernelStack: %8lu kB\n" +#ifdef CONFIG_SHADOW_CALL_STACK + "Node %d ShadowCallStack:%8lu kB\n" #endif - , - nid, K(node_page_state(nid, NR_FILE_DIRTY)), - nid, K(node_page_state(nid, NR_WRITEBACK)), - nid, K(node_page_state(nid, NR_FILE_PAGES)), - nid, K(node_page_state(nid, NR_FILE_MAPPED)), + "Node %d PageTables: %8lu kB\n" + "Node %d SecPageTables: %8lu kB\n" + "Node %d NFS_Unstable: %8lu kB\n" + "Node %d Bounce: %8lu kB\n" + "Node %d WritebackTmp: %8lu kB\n" + "Node %d KReclaimable: %8lu kB\n" + "Node %d Slab: %8lu kB\n" + "Node %d SReclaimable: %8lu kB\n" + "Node %d SUnreclaim: %8lu kB\n" #ifdef CONFIG_TRANSPARENT_HUGEPAGE - nid, K(node_page_state(nid, NR_ANON_PAGES) - + node_page_state(nid, NR_ANON_TRANSPARENT_HUGEPAGES) * - HPAGE_PMD_NR), -#else - nid, K(node_page_state(nid, NR_ANON_PAGES)), + "Node %d AnonHugePages: %8lu kB\n" + "Node %d ShmemHugePages: %8lu kB\n" + "Node %d ShmemPmdMapped: %8lu kB\n" + "Node %d FileHugePages: %8lu kB\n" + "Node %d FilePmdMapped: %8lu kB\n" +#endif +#ifdef CONFIG_UNACCEPTED_MEMORY + "Node %d Unaccepted: %8lu kB\n" +#endif + , + nid, K(node_page_state(pgdat, NR_FILE_DIRTY)), + nid, K(node_page_state(pgdat, NR_WRITEBACK)), + nid, K(node_page_state(pgdat, NR_FILE_PAGES)), + nid, K(node_page_state(pgdat, NR_FILE_MAPPED)), + nid, K(node_page_state(pgdat, NR_ANON_MAPPED)), + nid, K(i.sharedram), + nid, node_page_state(pgdat, NR_KERNEL_STACK_KB), +#ifdef CONFIG_SHADOW_CALL_STACK + nid, node_page_state(pgdat, NR_KERNEL_SCS_KB), #endif - nid, K(node_page_state(nid, NR_SHMEM)), - nid, node_page_state(nid, NR_KERNEL_STACK) * - THREAD_SIZE / 1024, - nid, K(node_page_state(nid, NR_PAGETABLE)), - nid, K(node_page_state(nid, NR_UNSTABLE_NFS)), - nid, K(node_page_state(nid, NR_BOUNCE)), - nid, K(node_page_state(nid, NR_WRITEBACK_TEMP)), - nid, K(node_page_state(nid, NR_SLAB_RECLAIMABLE) + - node_page_state(nid, NR_SLAB_UNRECLAIMABLE)), - nid, K(node_page_state(nid, NR_SLAB_RECLAIMABLE)), + nid, K(node_page_state(pgdat, NR_PAGETABLE)), + nid, K(node_page_state(pgdat, NR_SECONDARY_PAGETABLE)), + nid, 0UL, + nid, 0UL, + nid, 0UL, + nid, K(sreclaimable + + node_page_state(pgdat, NR_KERNEL_MISC_RECLAIMABLE)), + nid, K(sreclaimable + sunreclaimable), + nid, K(sreclaimable), + nid, K(sunreclaimable) #ifdef CONFIG_TRANSPARENT_HUGEPAGE - nid, K(node_page_state(nid, NR_SLAB_UNRECLAIMABLE)) - , nid, - K(node_page_state(nid, NR_ANON_TRANSPARENT_HUGEPAGES) * - HPAGE_PMD_NR)); -#else - nid, K(node_page_state(nid, NR_SLAB_UNRECLAIMABLE))); + , + nid, K(node_page_state(pgdat, NR_ANON_THPS)), + nid, K(node_page_state(pgdat, NR_SHMEM_THPS)), + nid, K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED)), + nid, K(node_page_state(pgdat, NR_FILE_THPS)), + nid, K(node_page_state(pgdat, NR_FILE_PMDMAPPED)) #endif - n += hugetlb_report_node_meminfo(nid, buf + n); - return n; +#ifdef CONFIG_UNACCEPTED_MEMORY + , + nid, K(sum_zone_node_page_state(nid, NR_UNACCEPTED)) +#endif + ); + len += hugetlb_report_node_meminfo(buf, len, nid); + return len; } #undef K -static DEVICE_ATTR(meminfo, S_IRUGO, node_read_meminfo, NULL); +static DEVICE_ATTR(meminfo, 0444, node_read_meminfo, NULL); static ssize_t node_read_numastat(struct device *dev, - struct device_attribute *attr, char *buf) + struct device_attribute *attr, char *buf) { - return sprintf(buf, - "numa_hit %lu\n" - "numa_miss %lu\n" - "numa_foreign %lu\n" - "interleave_hit %lu\n" - "local_node %lu\n" - "other_node %lu\n", - node_page_state(dev->id, NUMA_HIT), - node_page_state(dev->id, NUMA_MISS), - node_page_state(dev->id, NUMA_FOREIGN), - node_page_state(dev->id, NUMA_INTERLEAVE_HIT), - node_page_state(dev->id, NUMA_LOCAL), - node_page_state(dev->id, NUMA_OTHER)); -} -static DEVICE_ATTR(numastat, S_IRUGO, node_read_numastat, NULL); + fold_vm_numa_events(); + return sysfs_emit(buf, + "numa_hit %lu\n" + "numa_miss %lu\n" + "numa_foreign %lu\n" + "interleave_hit %lu\n" + "local_node %lu\n" + "other_node %lu\n", + sum_zone_numa_event_state(dev->id, NUMA_HIT), + sum_zone_numa_event_state(dev->id, NUMA_MISS), + sum_zone_numa_event_state(dev->id, NUMA_FOREIGN), + sum_zone_numa_event_state(dev->id, NUMA_INTERLEAVE_HIT), + sum_zone_numa_event_state(dev->id, NUMA_LOCAL), + sum_zone_numa_event_state(dev->id, NUMA_OTHER)); +} +static DEVICE_ATTR(numastat, 0444, node_read_numastat, NULL); static ssize_t node_read_vmstat(struct device *dev, struct device_attribute *attr, char *buf) { int nid = dev->id; + struct pglist_data *pgdat = NODE_DATA(nid); int i; - int n = 0; + int len = 0; for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) - n += sprintf(buf+n, "%s %lu\n", vmstat_text[i], - node_page_state(nid, i)); + len += sysfs_emit_at(buf, len, "%s %lu\n", + zone_stat_name(i), + sum_zone_node_page_state(nid, i)); - return n; +#ifdef CONFIG_NUMA + fold_vm_numa_events(); + for (i = 0; i < NR_VM_NUMA_EVENT_ITEMS; i++) + len += sysfs_emit_at(buf, len, "%s %lu\n", + numa_stat_name(i), + sum_zone_numa_event_state(nid, i)); + +#endif + for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { + unsigned long pages = node_page_state_pages(pgdat, i); + + if (vmstat_item_print_in_thp(i)) + pages /= HPAGE_PMD_NR; + len += sysfs_emit_at(buf, len, "%s %lu\n", node_stat_name(i), + pages); + } + + return len; } -static DEVICE_ATTR(vmstat, S_IRUGO, node_read_vmstat, NULL); +static DEVICE_ATTR(vmstat, 0444, node_read_vmstat, NULL); static ssize_t node_read_distance(struct device *dev, - struct device_attribute *attr, char * buf) + struct device_attribute *attr, char *buf) { int nid = dev->id; int len = 0; @@ -204,126 +631,49 @@ static ssize_t node_read_distance(struct device *dev, */ BUILD_BUG_ON(MAX_NUMNODES * 4 > PAGE_SIZE); - for_each_online_node(i) - len += sprintf(buf + len, "%s%d", i ? " " : "", node_distance(nid, i)); + for_each_online_node(i) { + len += sysfs_emit_at(buf, len, "%s%d", + i ? " " : "", node_distance(nid, i)); + } - len += sprintf(buf + len, "\n"); + len += sysfs_emit_at(buf, len, "\n"); return len; } -static DEVICE_ATTR(distance, S_IRUGO, node_read_distance, NULL); - -#ifdef CONFIG_HUGETLBFS -/* - * hugetlbfs per node attributes registration interface: - * When/if hugetlb[fs] subsystem initializes [sometime after this module], - * it will register its per node attributes for all online nodes with - * memory. It will also call register_hugetlbfs_with_node(), below, to - * register its attribute registration functions with this node driver. - * Once these hooks have been initialized, the node driver will call into - * the hugetlb module to [un]register attributes for hot-plugged nodes. - */ -static node_registration_func_t __hugetlb_register_node; -static node_registration_func_t __hugetlb_unregister_node; +static DEVICE_ATTR(distance, 0444, node_read_distance, NULL); -static inline bool hugetlb_register_node(struct node *node) -{ - if (__hugetlb_register_node && - node_state(node->dev.id, N_MEMORY)) { - __hugetlb_register_node(node); - return true; - } - return false; -} +static struct attribute *node_dev_attrs[] = { + &dev_attr_meminfo.attr, + &dev_attr_numastat.attr, + &dev_attr_distance.attr, + &dev_attr_vmstat.attr, + NULL +}; -static inline void hugetlb_unregister_node(struct node *node) -{ - if (__hugetlb_unregister_node) - __hugetlb_unregister_node(node); -} +static const struct bin_attribute *node_dev_bin_attrs[] = { + &bin_attr_cpumap, + &bin_attr_cpulist, + NULL +}; -void register_hugetlbfs_with_node(node_registration_func_t doregister, - node_registration_func_t unregister) -{ - __hugetlb_register_node = doregister; - __hugetlb_unregister_node = unregister; -} -#else -static inline void hugetlb_register_node(struct node *node) {} +static const struct attribute_group node_dev_group = { + .attrs = node_dev_attrs, + .bin_attrs = node_dev_bin_attrs, +}; -static inline void hugetlb_unregister_node(struct node *node) {} +static const struct attribute_group *node_dev_groups[] = { + &node_dev_group, +#ifdef CONFIG_HAVE_ARCH_NODE_DEV_GROUP + &arch_node_dev_group, #endif - -static void node_device_release(struct device *dev) -{ - struct node *node = to_node(dev); - -#if defined(CONFIG_MEMORY_HOTPLUG_SPARSE) && defined(CONFIG_HUGETLBFS) - /* - * We schedule the work only when a memory section is - * onlined/offlined on this node. When we come here, - * all the memory on this node has been offlined, - * so we won't enqueue new work to this work. - * - * The work is using node->node_work, so we should - * flush work before freeing the memory. - */ - flush_work(&node->node_work); +#ifdef CONFIG_MEMORY_FAILURE + &memory_failure_attr_group, #endif - kfree(node); -} - -/* - * register_node - Setup a sysfs device for a node. - * @num - Node number to use when creating the device. - * - * Initialize and register the node device. - */ -static int register_node(struct node *node, int num, struct node *parent) -{ - int error; - - node->dev.id = num; - node->dev.bus = &node_subsys; - node->dev.release = node_device_release; - error = device_register(&node->dev); - - if (!error){ - device_create_file(&node->dev, &dev_attr_cpumap); - device_create_file(&node->dev, &dev_attr_cpulist); - device_create_file(&node->dev, &dev_attr_meminfo); - device_create_file(&node->dev, &dev_attr_numastat); - device_create_file(&node->dev, &dev_attr_distance); - device_create_file(&node->dev, &dev_attr_vmstat); - - scan_unevictable_register_node(node); - - hugetlb_register_node(node); - - compaction_register_node(node); - } - return error; -} + NULL +}; -/** - * unregister_node - unregister a node device - * @node: node going away - * - * Unregisters a node device @node. All the devices on the node must be - * unregistered before calling this function. - */ -void unregister_node(struct node *node) +static void node_device_release(struct device *dev) { - device_remove_file(&node->dev, &dev_attr_cpumap); - device_remove_file(&node->dev, &dev_attr_cpulist); - device_remove_file(&node->dev, &dev_attr_meminfo); - device_remove_file(&node->dev, &dev_attr_numastat); - device_remove_file(&node->dev, &dev_attr_distance); - device_remove_file(&node->dev, &dev_attr_vmstat); - - scan_unevictable_unregister_node(node); - hugetlb_unregister_node(node); /* no-op, if memoryless node */ - - device_unregister(&node->dev); + kfree(to_node(dev)); } struct node *node_devices[MAX_NUMNODES]; @@ -354,6 +704,56 @@ int register_cpu_under_node(unsigned int cpu, unsigned int nid) kobject_name(&node_devices[nid]->dev.kobj)); } +/** + * register_memory_node_under_compute_node - link memory node to its compute + * node for a given access class. + * @mem_nid: Memory node number + * @cpu_nid: Cpu node number + * @access: Access class to register + * + * Description: + * For use with platforms that may have separate memory and compute nodes. + * This function will export node relationships linking which memory + * initiator nodes can access memory targets at a given ranked access + * class. + */ +int register_memory_node_under_compute_node(unsigned int mem_nid, + unsigned int cpu_nid, + enum access_coordinate_class access) +{ + struct node *init_node, *targ_node; + struct node_access_nodes *initiator, *target; + int ret; + + if (!node_online(cpu_nid) || !node_online(mem_nid)) + return -ENODEV; + + init_node = node_devices[cpu_nid]; + targ_node = node_devices[mem_nid]; + initiator = node_init_node_access(init_node, access); + target = node_init_node_access(targ_node, access); + if (!initiator || !target) + return -ENOMEM; + + ret = sysfs_add_link_to_group(&initiator->dev.kobj, "targets", + &targ_node->dev.kobj, + dev_name(&targ_node->dev)); + if (ret) + return ret; + + ret = sysfs_add_link_to_group(&target->dev.kobj, "initiators", + &init_node->dev.kobj, + dev_name(&init_node->dev)); + if (ret) + goto err; + + return 0; + err: + sysfs_remove_link_from_group(&initiator->dev.kobj, "targets", + dev_name(&targ_node->dev)); + return ret; +} + int unregister_cpu_under_node(unsigned int cpu, unsigned int nid) { struct device *obj; @@ -373,239 +773,160 @@ int unregister_cpu_under_node(unsigned int cpu, unsigned int nid) return 0; } -#ifdef CONFIG_MEMORY_HOTPLUG_SPARSE -#define page_initialized(page) (page->lru.next) - -static int get_nid_for_pfn(unsigned long pfn) -{ - struct page *page; - - if (!pfn_valid_within(pfn)) - return -1; - page = pfn_to_page(pfn); - if (!page_initialized(page)) - return -1; - return pfn_to_nid(pfn); -} - -/* register memory section under specified node if it spans that node */ -int register_mem_sect_under_node(struct memory_block *mem_blk, int nid) +#ifdef CONFIG_MEMORY_HOTPLUG +static void do_register_memory_block_under_node(int nid, + struct memory_block *mem_blk) { int ret; - unsigned long pfn, sect_start_pfn, sect_end_pfn; - - if (!mem_blk) - return -EFAULT; - if (!node_online(nid)) - return 0; - - sect_start_pfn = section_nr_to_pfn(mem_blk->start_section_nr); - sect_end_pfn = section_nr_to_pfn(mem_blk->end_section_nr); - sect_end_pfn += PAGES_PER_SECTION - 1; - for (pfn = sect_start_pfn; pfn <= sect_end_pfn; pfn++) { - int page_nid; - page_nid = get_nid_for_pfn(pfn); - if (page_nid < 0) - continue; - if (page_nid != nid) - continue; - ret = sysfs_create_link_nowarn(&node_devices[nid]->dev.kobj, - &mem_blk->dev.kobj, - kobject_name(&mem_blk->dev.kobj)); - if (ret) - return ret; + ret = sysfs_create_link_nowarn(&node_devices[nid]->dev.kobj, + &mem_blk->dev.kobj, + kobject_name(&mem_blk->dev.kobj)); + if (ret && ret != -EEXIST) + dev_err_ratelimited(&node_devices[nid]->dev, + "can't create link to %s in sysfs (%d)\n", + kobject_name(&mem_blk->dev.kobj), ret); - return sysfs_create_link_nowarn(&mem_blk->dev.kobj, + ret = sysfs_create_link_nowarn(&mem_blk->dev.kobj, &node_devices[nid]->dev.kobj, kobject_name(&node_devices[nid]->dev.kobj)); - } - /* mem section does not span the specified node */ - return 0; + if (ret && ret != -EEXIST) + dev_err_ratelimited(&mem_blk->dev, + "can't create link to %s in sysfs (%d)\n", + kobject_name(&node_devices[nid]->dev.kobj), + ret); } -/* unregister memory section under all nodes that it spans */ -int unregister_mem_sect_under_nodes(struct memory_block *mem_blk, - unsigned long phys_index) +/* + * During hotplug we know that all pages in the memory block belong to the same + * node. + */ +static int register_mem_block_under_node_hotplug(struct memory_block *mem_blk, + void *arg) { - NODEMASK_ALLOC(nodemask_t, unlinked_nodes, GFP_KERNEL); - unsigned long pfn, sect_start_pfn, sect_end_pfn; - - if (!mem_blk) { - NODEMASK_FREE(unlinked_nodes); - return -EFAULT; - } - if (!unlinked_nodes) - return -ENOMEM; - nodes_clear(*unlinked_nodes); - - sect_start_pfn = section_nr_to_pfn(phys_index); - sect_end_pfn = sect_start_pfn + PAGES_PER_SECTION - 1; - for (pfn = sect_start_pfn; pfn <= sect_end_pfn; pfn++) { - int nid; + int nid = *(int *)arg; - nid = get_nid_for_pfn(pfn); - if (nid < 0) - continue; - if (!node_online(nid)) - continue; - if (node_test_and_set(nid, *unlinked_nodes)) - continue; - sysfs_remove_link(&node_devices[nid]->dev.kobj, - kobject_name(&mem_blk->dev.kobj)); - sysfs_remove_link(&mem_blk->dev.kobj, - kobject_name(&node_devices[nid]->dev.kobj)); - } - NODEMASK_FREE(unlinked_nodes); + do_register_memory_block_under_node(nid, mem_blk); return 0; } -static int link_mem_sections(int nid) -{ - unsigned long start_pfn = NODE_DATA(nid)->node_start_pfn; - unsigned long end_pfn = start_pfn + NODE_DATA(nid)->node_spanned_pages; - unsigned long pfn; - struct memory_block *mem_blk = NULL; - int err = 0; - - for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) { - unsigned long section_nr = pfn_to_section_nr(pfn); - struct mem_section *mem_sect; - int ret; - - if (!present_section_nr(section_nr)) - continue; - mem_sect = __nr_to_section(section_nr); - - /* same memblock ? */ - if (mem_blk) - if ((section_nr >= mem_blk->start_section_nr) && - (section_nr <= mem_blk->end_section_nr)) - continue; - - mem_blk = find_memory_block_hinted(mem_sect, mem_blk); - - ret = register_mem_sect_under_node(mem_blk, nid); - if (!err) - err = ret; - - /* discard ref obtained in find_memory_block() */ - } - - if (mem_blk) - kobject_put(&mem_blk->dev.kobj); - return err; -} - -#ifdef CONFIG_HUGETLBFS /* - * Handle per node hstate attribute [un]registration on transistions - * to/from memoryless state. + * Unregister a memory block device under the node it spans. Memory blocks + * with multiple nodes cannot be offlined and therefore also never be removed. */ -static void node_hugetlb_work(struct work_struct *work) +void unregister_memory_block_under_nodes(struct memory_block *mem_blk) { - struct node *node = container_of(work, struct node, node_work); + if (mem_blk->nid == NUMA_NO_NODE) + return; - /* - * We only get here when a node transitions to/from memoryless state. - * We can detect which transition occurred by examining whether the - * node has memory now. hugetlb_register_node() already check this - * so we try to register the attributes. If that fails, then the - * node has transitioned to memoryless, try to unregister the - * attributes. - */ - if (!hugetlb_register_node(node)) - hugetlb_unregister_node(node); + sysfs_remove_link(&node_devices[mem_blk->nid]->dev.kobj, + kobject_name(&mem_blk->dev.kobj)); + sysfs_remove_link(&mem_blk->dev.kobj, + kobject_name(&node_devices[mem_blk->nid]->dev.kobj)); } -static void init_node_hugetlb_work(int nid) +/* register all memory blocks under the corresponding nodes */ +static void register_memory_blocks_under_nodes(void) { - INIT_WORK(&node_devices[nid]->node_work, node_hugetlb_work); -} + struct memblock_region *r; -static int node_memory_callback(struct notifier_block *self, - unsigned long action, void *arg) -{ - struct memory_notify *mnb = arg; - int nid = mnb->status_change_nid; + for_each_mem_region(r) { + const unsigned long start_block_id = phys_to_block_id(r->base); + const unsigned long end_block_id = phys_to_block_id(r->base + r->size - 1); + const int nid = memblock_get_region_node(r); + unsigned long block_id; - switch (action) { - case MEM_ONLINE: - case MEM_OFFLINE: - /* - * offload per node hstate [un]registration to a work thread - * when transitioning to/from memoryless state. - */ - if (nid != NUMA_NO_NODE) - schedule_work(&node_devices[nid]->node_work); - break; + if (!node_online(nid)) + continue; - case MEM_GOING_ONLINE: - case MEM_GOING_OFFLINE: - case MEM_CANCEL_ONLINE: - case MEM_CANCEL_OFFLINE: - default: - break; - } + for (block_id = start_block_id; block_id <= end_block_id; block_id++) { + struct memory_block *mem; - return NOTIFY_OK; -} -#endif /* CONFIG_HUGETLBFS */ -#else /* !CONFIG_MEMORY_HOTPLUG_SPARSE */ + mem = find_memory_block_by_id(block_id); + if (!mem) + continue; -static int link_mem_sections(int nid) { return 0; } -#endif /* CONFIG_MEMORY_HOTPLUG_SPARSE */ + memory_block_add_nid_early(mem, nid); + do_register_memory_block_under_node(nid, mem); + put_device(&mem->dev); + } -#if !defined(CONFIG_MEMORY_HOTPLUG_SPARSE) || \ - !defined(CONFIG_HUGETLBFS) -static inline int node_memory_callback(struct notifier_block *self, - unsigned long action, void *arg) -{ - return NOTIFY_OK; + } } -static void init_node_hugetlb_work(int nid) { } - -#endif +void register_memory_blocks_under_node_hotplug(int nid, unsigned long start_pfn, + unsigned long end_pfn) +{ + walk_memory_blocks(PFN_PHYS(start_pfn), PFN_PHYS(end_pfn - start_pfn), + (void *)&nid, register_mem_block_under_node_hotplug); + return; +} +#endif /* CONFIG_MEMORY_HOTPLUG */ -int register_one_node(int nid) +/** + * register_node - Initialize and register the node device. + * @nid: Node number to use when creating the device. + * + * Return: 0 on success, -errno otherwise + */ +int register_node(int nid) { - int error = 0; + int error; int cpu; + struct node *node; - if (node_online(nid)) { - int p_node = parent_node(nid); - struct node *parent = NULL; - - if (p_node != nid) - parent = node_devices[p_node]; + node = kzalloc(sizeof(struct node), GFP_KERNEL); + if (!node) + return -ENOMEM; - node_devices[nid] = kzalloc(sizeof(struct node), GFP_KERNEL); - if (!node_devices[nid]) - return -ENOMEM; + INIT_LIST_HEAD(&node->access_list); - error = register_node(node_devices[nid], nid, parent); + node->dev.id = nid; + node->dev.bus = &node_subsys; + node->dev.release = node_device_release; + node->dev.groups = node_dev_groups; - /* link cpu under this node */ - for_each_present_cpu(cpu) { - if (cpu_to_node(cpu) == nid) - register_cpu_under_node(cpu, nid); - } + error = device_register(&node->dev); + if (error) { + put_device(&node->dev); + return error; + } - /* link memory sections under this node */ - error = link_mem_sections(nid); + node_devices[nid] = node; + hugetlb_register_node(node); + compaction_register_node(node); + reclaim_register_node(node); - /* initialize work queue for memory hot plug */ - init_node_hugetlb_work(nid); + /* link cpu under this node */ + for_each_present_cpu(cpu) { + if (cpu_to_node(cpu) == nid) + register_cpu_under_node(cpu, nid); } - return error; + node_init_caches(nid); + return error; } - -void unregister_one_node(int nid) +/** + * unregister_node - unregister a node device + * @nid: nid of the node going away + * + * Unregisters the node device at node id @nid. All the devices on the + * node must be unregistered before calling this function. + */ +void unregister_node(int nid) { - unregister_node(node_devices[nid]); + struct node *node = node_devices[nid]; + + if (!node) + return; + + hugetlb_unregister_node(node); + compaction_unregister_node(node); + reclaim_unregister_node(node); + node_remove_accesses(node); + node_remove_caches(node); + device_unregister(&node->dev); node_devices[nid] = NULL; } @@ -613,16 +934,6 @@ void unregister_one_node(int nid) * node states attributes */ -static ssize_t print_nodes_state(enum node_states state, char *buf) -{ - int n; - - n = nodelist_scnprintf(buf, PAGE_SIZE-2, node_states[state]); - buf[n++] = '\n'; - buf[n] = '\0'; - return n; -} - struct node_attr { struct device_attribute attr; enum node_states state; @@ -632,7 +943,9 @@ static ssize_t show_node_state(struct device *dev, struct device_attribute *attr, char *buf) { struct node_attr *na = container_of(attr, struct node_attr, attr); - return print_nodes_state(na->state, buf); + + return sysfs_emit(buf, "%*pbl\n", + nodemask_pr_args(&node_states[na->state])); } #define _NODE_ATTR(name, state) \ @@ -645,10 +958,10 @@ static struct node_attr node_state_attr[] = { #ifdef CONFIG_HIGHMEM [N_HIGH_MEMORY] = _NODE_ATTR(has_high_memory, N_HIGH_MEMORY), #endif -#ifdef CONFIG_MOVABLE_NODE [N_MEMORY] = _NODE_ATTR(has_memory, N_MEMORY), -#endif [N_CPU] = _NODE_ATTR(has_cpu, N_CPU), + [N_GENERIC_INITIATOR] = _NODE_ATTR(has_generic_initiator, + N_GENERIC_INITIATOR), }; static struct attribute *node_state_attrs[] = { @@ -658,14 +971,13 @@ static struct attribute *node_state_attrs[] = { #ifdef CONFIG_HIGHMEM &node_state_attr[N_HIGH_MEMORY].attr.attr, #endif -#ifdef CONFIG_MOVABLE_NODE &node_state_attr[N_MEMORY].attr.attr, -#endif &node_state_attr[N_CPU].attr.attr, + &node_state_attr[N_GENERIC_INITIATOR].attr.attr, NULL }; -static struct attribute_group memory_root_attr_group = { +static const struct attribute_group memory_root_attr_group = { .attrs = node_state_attrs, }; @@ -674,27 +986,26 @@ static const struct attribute_group *cpu_root_attr_groups[] = { NULL, }; -#define NODE_CALLBACK_PRI 2 /* lower than SLAB */ -static int __init register_node_type(void) +void __init node_dev_init(void) { - int ret; + int ret, i; BUILD_BUG_ON(ARRAY_SIZE(node_state_attr) != NR_NODE_STATES); BUILD_BUG_ON(ARRAY_SIZE(node_state_attrs)-1 != NR_NODE_STATES); ret = subsys_system_register(&node_subsys, cpu_root_attr_groups); - if (!ret) { - static struct notifier_block node_memory_callback_nb = { - .notifier_call = node_memory_callback, - .priority = NODE_CALLBACK_PRI, - }; - register_hotmemory_notifier(&node_memory_callback_nb); - } + if (ret) + panic("%s() failed to register subsystem: %d\n", __func__, ret); /* - * Note: we're not going to unregister the node class if we fail - * to register the node state class attribute files. + * Create all node devices, which will properly link the node + * to already created cpu devices. */ - return ret; + for_each_online_node(i) { + ret = register_node(i); + if (ret) + panic("%s() failed to add node: %d\n", __func__, ret); + } + + register_memory_blocks_under_nodes(); } -postcore_initcall(register_node_type); diff --git a/drivers/base/physical_location.c b/drivers/base/physical_location.c new file mode 100644 index 000000000000..a5539e294d4d --- /dev/null +++ b/drivers/base/physical_location.c @@ -0,0 +1,145 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Device physical location support + * + * Author: Won Chung <wonchung@google.com> + */ + +#include <linux/acpi.h> +#include <linux/sysfs.h> +#include <linux/string_choices.h> + +#include "physical_location.h" + +bool dev_add_physical_location(struct device *dev) +{ + struct acpi_pld_info *pld; + + if (!has_acpi_companion(dev)) + return false; + + if (!acpi_get_physical_device_location(ACPI_HANDLE(dev), &pld)) + return false; + + dev->physical_location = + kzalloc(sizeof(*dev->physical_location), GFP_KERNEL); + if (!dev->physical_location) { + ACPI_FREE(pld); + return false; + } + + dev->physical_location->panel = pld->panel; + dev->physical_location->vertical_position = pld->vertical_position; + dev->physical_location->horizontal_position = pld->horizontal_position; + dev->physical_location->dock = pld->dock; + dev->physical_location->lid = pld->lid; + + ACPI_FREE(pld); + return true; +} + +static ssize_t panel_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + const char *panel; + + switch (dev->physical_location->panel) { + case DEVICE_PANEL_TOP: + panel = "top"; + break; + case DEVICE_PANEL_BOTTOM: + panel = "bottom"; + break; + case DEVICE_PANEL_LEFT: + panel = "left"; + break; + case DEVICE_PANEL_RIGHT: + panel = "right"; + break; + case DEVICE_PANEL_FRONT: + panel = "front"; + break; + case DEVICE_PANEL_BACK: + panel = "back"; + break; + default: + panel = "unknown"; + } + return sysfs_emit(buf, "%s\n", panel); +} +static DEVICE_ATTR_RO(panel); + +static ssize_t vertical_position_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + const char *vertical_position; + + switch (dev->physical_location->vertical_position) { + case DEVICE_VERT_POS_UPPER: + vertical_position = "upper"; + break; + case DEVICE_VERT_POS_CENTER: + vertical_position = "center"; + break; + case DEVICE_VERT_POS_LOWER: + vertical_position = "lower"; + break; + default: + vertical_position = "unknown"; + } + return sysfs_emit(buf, "%s\n", vertical_position); +} +static DEVICE_ATTR_RO(vertical_position); + +static ssize_t horizontal_position_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + const char *horizontal_position; + + switch (dev->physical_location->horizontal_position) { + case DEVICE_HORI_POS_LEFT: + horizontal_position = "left"; + break; + case DEVICE_HORI_POS_CENTER: + horizontal_position = "center"; + break; + case DEVICE_HORI_POS_RIGHT: + horizontal_position = "right"; + break; + default: + horizontal_position = "unknown"; + } + return sysfs_emit(buf, "%s\n", horizontal_position); +} +static DEVICE_ATTR_RO(horizontal_position); + +static ssize_t dock_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + return sysfs_emit(buf, "%s\n", + str_yes_no(dev->physical_location->dock)); +} +static DEVICE_ATTR_RO(dock); + +static ssize_t lid_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + return sysfs_emit(buf, "%s\n", + str_yes_no(dev->physical_location->lid)); +} +static DEVICE_ATTR_RO(lid); + +static struct attribute *dev_attr_physical_location[] = { + &dev_attr_panel.attr, + &dev_attr_vertical_position.attr, + &dev_attr_horizontal_position.attr, + &dev_attr_dock.attr, + &dev_attr_lid.attr, + NULL, +}; + +const struct attribute_group dev_attr_physical_location_group = { + .name = "physical_location", + .attrs = dev_attr_physical_location, +}; + diff --git a/drivers/base/physical_location.h b/drivers/base/physical_location.h new file mode 100644 index 000000000000..3f3f61307998 --- /dev/null +++ b/drivers/base/physical_location.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Device physical location support + * + * Author: Won Chung <wonchung@google.com> + */ + +#include <linux/device.h> + +#ifdef CONFIG_ACPI +bool dev_add_physical_location(struct device *dev); +extern const struct attribute_group dev_attr_physical_location_group; +#else +static inline bool dev_add_physical_location(struct device *dev) { return false; }; +static const struct attribute_group dev_attr_physical_location_group = {}; +#endif diff --git a/drivers/base/pinctrl.c b/drivers/base/pinctrl.c index 5fb74b43848e..c22864458511 100644 --- a/drivers/base/pinctrl.c +++ b/drivers/base/pinctrl.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Driver core interface to the pinctrl subsystem. * @@ -6,8 +7,6 @@ * Based on bits of regulator core, gpio core and clk core * * Author: Linus Walleij <linus.walleij@linaro.org> - * - * License terms: GNU General Public License (GPL) version 2 */ #include <linux/device.h> @@ -23,6 +22,9 @@ int pinctrl_bind_pins(struct device *dev) { int ret; + if (dev->of_node_reused) + return 0; + dev->pins = devm_kzalloc(dev, sizeof(*(dev->pins)), GFP_KERNEL); if (!dev->pins) return -ENOMEM; @@ -42,9 +44,20 @@ int pinctrl_bind_pins(struct device *dev) goto cleanup_get; } - ret = pinctrl_select_state(dev->pins->p, dev->pins->default_state); + dev->pins->init_state = pinctrl_lookup_state(dev->pins->p, + PINCTRL_STATE_INIT); + if (IS_ERR(dev->pins->init_state)) { + /* Not supplying this state is perfectly legal */ + dev_dbg(dev, "no init pinctrl state\n"); + + ret = pinctrl_select_state(dev->pins->p, + dev->pins->default_state); + } else { + ret = pinctrl_select_state(dev->pins->p, dev->pins->init_state); + } + if (ret) { - dev_dbg(dev, "failed to activate default pinctrl state\n"); + dev_dbg(dev, "failed to activate initial pinctrl state\n"); goto cleanup_get; } @@ -80,9 +93,13 @@ cleanup_alloc: devm_kfree(dev, dev->pins); dev->pins = NULL; - /* Only return deferrals */ - if (ret != -EPROBE_DEFER) - ret = 0; + /* Return deferrals */ + if (ret == -EPROBE_DEFER) + return ret; + /* Return serious errors */ + if (ret == -EINVAL) + return ret; + /* We ignore errors like -ENOENT meaning no pinctrl state */ - return ret; + return 0; } diff --git a/drivers/base/platform-msi.c b/drivers/base/platform-msi.c new file mode 100644 index 000000000000..70db08f3ac6f --- /dev/null +++ b/drivers/base/platform-msi.c @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * MSI framework for platform devices + * + * Copyright (C) 2015 ARM Limited, All Rights Reserved. + * Author: Marc Zyngier <marc.zyngier@arm.com> + * Copyright (C) 2022 Linutronix GmbH + */ + +#include <linux/device.h> +#include <linux/irqdomain.h> +#include <linux/msi.h> + +/* + * This indirection can go when platform_device_msi_init_and_alloc_irqs() + * is switched to a proper irq_chip::irq_write_msi_msg() callback. Keep it + * simple for now. + */ +static void platform_msi_write_msi_msg(struct irq_data *d, struct msi_msg *msg) +{ + irq_write_msi_msg_t cb = d->chip_data; + + cb(irq_data_get_msi_desc(d), msg); +} + +static void platform_msi_set_desc(msi_alloc_info_t *arg, struct msi_desc *desc) +{ + arg->desc = desc; + arg->hwirq = desc->msi_index; +} + +static const struct msi_domain_template platform_msi_template = { + .chip = { + .name = "pMSI", + .irq_mask = irq_chip_mask_parent, + .irq_unmask = irq_chip_unmask_parent, + .irq_write_msi_msg = platform_msi_write_msi_msg, + /* The rest is filled in by the platform MSI parent */ + }, + + .ops = { + .set_desc = platform_msi_set_desc, + }, + + .info = { + .bus_token = DOMAIN_BUS_DEVICE_MSI, + }, +}; + +/** + * platform_device_msi_init_and_alloc_irqs - Initialize platform device MSI + * and allocate interrupts for @dev + * @dev: The device for which to allocate interrupts + * @nvec: The number of interrupts to allocate + * @write_msi_msg: Callback to write an interrupt message for @dev + * + * Returns: + * Zero for success, or an error code in case of failure + * + * This creates a MSI domain on @dev which has @dev->msi.domain as + * parent. The parent domain sets up the new domain. The domain has + * a fixed size of @nvec. The domain is managed by devres and will + * be removed when the device is removed. + * + * Note: For migration purposes this falls back to the original platform_msi code + * up to the point where all platforms have been converted to the MSI + * parent model. + */ +int platform_device_msi_init_and_alloc_irqs(struct device *dev, unsigned int nvec, + irq_write_msi_msg_t write_msi_msg) +{ + struct irq_domain *domain = dev->msi.domain; + + if (!domain || !write_msi_msg) + return -EINVAL; + + /* + * @write_msi_msg is stored in the resulting msi_domain_info::data. + * The underlying domain creation mechanism will assign that + * callback to the resulting irq chip. + */ + if (!msi_create_device_irq_domain(dev, MSI_DEFAULT_DOMAIN, + &platform_msi_template, + nvec, NULL, write_msi_msg)) + return -ENODEV; + + return msi_domain_alloc_irqs_range(dev, MSI_DEFAULT_DOMAIN, 0, nvec - 1); +} +EXPORT_SYMBOL_GPL(platform_device_msi_init_and_alloc_irqs); + +/** + * platform_device_msi_free_irqs_all - Free all interrupts for @dev + * @dev: The device for which to free interrupts + */ +void platform_device_msi_free_irqs_all(struct device *dev) +{ + msi_domain_free_irqs_all(dev, MSI_DEFAULT_DOMAIN); + msi_remove_device_irq_domain(dev, MSI_DEFAULT_DOMAIN); +} +EXPORT_SYMBOL_GPL(platform_device_msi_free_irqs_all); diff --git a/drivers/base/platform.c b/drivers/base/platform.c index 15789875128e..b45d41b018ca 100644 --- a/drivers/base/platform.c +++ b/drivers/base/platform.c @@ -1,27 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0 /* * platform.c - platform 'pseudo' bus for legacy devices * * Copyright (c) 2002-3 Patrick Mochel * Copyright (c) 2002-3 Open Source Development Labs * - * This file is released under the GPLv2 - * - * Please see Documentation/driver-model/platform.txt for more + * Please see Documentation/driver-api/driver-model/platform.rst for more * information. */ #include <linux/string.h> #include <linux/platform_device.h> #include <linux/of_device.h> +#include <linux/of_irq.h> #include <linux/module.h> #include <linux/init.h> +#include <linux/interrupt.h> +#include <linux/ioport.h> #include <linux/dma-mapping.h> -#include <linux/bootmem.h> +#include <linux/memblock.h> #include <linux/err.h> #include <linux/slab.h> #include <linux/pm_runtime.h> +#include <linux/pm_domain.h> #include <linux/idr.h> #include <linux/acpi.h> +#include <linux/clk/clk-conf.h> +#include <linux/limits.h> +#include <linux/property.h> +#include <linux/kmemleak.h> +#include <linux/types.h> +#include <linux/iommu.h> +#include <linux/dma-map-ops.h> #include "base.h" #include "power/power.h" @@ -35,34 +45,17 @@ struct device platform_bus = { EXPORT_SYMBOL_GPL(platform_bus); /** - * arch_setup_pdev_archdata - Allow manipulation of archdata before its used - * @pdev: platform device - * - * This is called before platform_device_add() such that any pdev_archdata may - * be setup before the platform_notifier is called. So if a user needs to - * manipulate any relevant information in the pdev_archdata they can do: - * - * platform_device_alloc() - * ... manipulate ... - * platform_device_add() - * - * And if they don't care they can just call platform_device_register() and - * everything will just work out. - */ -void __weak arch_setup_pdev_archdata(struct platform_device *pdev) -{ -} - -/** * platform_get_resource - get a resource for a device * @dev: platform device * @type: resource type * @num: resource index + * + * Return: a pointer to the resource or NULL on failure. */ struct resource *platform_get_resource(struct platform_device *dev, unsigned int type, unsigned int num) { - int i; + u32 i; for (i = 0; i < dev->num_resources; i++) { struct resource *r = &dev->resource[i]; @@ -74,27 +67,392 @@ struct resource *platform_get_resource(struct platform_device *dev, } EXPORT_SYMBOL_GPL(platform_get_resource); +struct resource *platform_get_mem_or_io(struct platform_device *dev, + unsigned int num) +{ + u32 i; + + for (i = 0; i < dev->num_resources; i++) { + struct resource *r = &dev->resource[i]; + + if ((resource_type(r) & (IORESOURCE_MEM|IORESOURCE_IO)) && num-- == 0) + return r; + } + return NULL; +} +EXPORT_SYMBOL_GPL(platform_get_mem_or_io); + +#ifdef CONFIG_HAS_IOMEM /** - * platform_get_irq - get an IRQ for a device - * @dev: platform device - * @num: IRQ number index + * devm_platform_get_and_ioremap_resource - call devm_ioremap_resource() for a + * platform device and get resource + * + * @pdev: platform device to use both for memory resource lookup as well as + * resource management + * @index: resource index + * @res: optional output parameter to store a pointer to the obtained resource. + * + * Return: a pointer to the remapped memory or an ERR_PTR() encoded error code + * on failure. */ -int platform_get_irq(struct platform_device *dev, unsigned int num) +void __iomem * +devm_platform_get_and_ioremap_resource(struct platform_device *pdev, + unsigned int index, struct resource **res) +{ + struct resource *r; + + r = platform_get_resource(pdev, IORESOURCE_MEM, index); + if (res) + *res = r; + return devm_ioremap_resource(&pdev->dev, r); +} +EXPORT_SYMBOL_GPL(devm_platform_get_and_ioremap_resource); + +/** + * devm_platform_ioremap_resource - call devm_ioremap_resource() for a platform + * device + * + * @pdev: platform device to use both for memory resource lookup as well as + * resource management + * @index: resource index + * + * Return: a pointer to the remapped memory or an ERR_PTR() encoded error code + * on failure. + */ +void __iomem *devm_platform_ioremap_resource(struct platform_device *pdev, + unsigned int index) +{ + return devm_platform_get_and_ioremap_resource(pdev, index, NULL); +} +EXPORT_SYMBOL_GPL(devm_platform_ioremap_resource); + +/** + * devm_platform_ioremap_resource_byname - call devm_ioremap_resource for + * a platform device, retrieve the + * resource by name + * + * @pdev: platform device to use both for memory resource lookup as well as + * resource management + * @name: name of the resource + * + * Return: a pointer to the remapped memory or an ERR_PTR() encoded error code + * on failure. + */ +void __iomem * +devm_platform_ioremap_resource_byname(struct platform_device *pdev, + const char *name) +{ + struct resource *res; + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, name); + return devm_ioremap_resource(&pdev->dev, res); +} +EXPORT_SYMBOL_GPL(devm_platform_ioremap_resource_byname); +#endif /* CONFIG_HAS_IOMEM */ + +static const struct cpumask *get_irq_affinity(struct platform_device *dev, + unsigned int num) +{ + const struct cpumask *mask = NULL; +#ifndef CONFIG_SPARC + struct fwnode_handle *fwnode = dev_fwnode(&dev->dev); + + if (is_of_node(fwnode)) + mask = of_irq_get_affinity(to_of_node(fwnode), num); + else if (is_acpi_device_node(fwnode)) + mask = acpi_irq_get_affinity(ACPI_HANDLE_FWNODE(fwnode), num); +#endif + + return mask ?: cpu_possible_mask; +} + +/** + * platform_get_irq_affinity - get an optional IRQ and its affinity for a device + * @dev: platform device + * @num: interrupt number index + * @affinity: optional cpumask pointer to get the affinity of a per-cpu interrupt + * + * Gets an interupt for a platform device. Device drivers should check the + * return value for errors so as to not pass a negative integer value to + * the request_irq() APIs. Optional affinity information is provided in the + * affinity pointer if available, and NULL otherwise. + * + * Return: non-zero interrupt number on success, negative error number on failure. + */ +int platform_get_irq_affinity(struct platform_device *dev, unsigned int num, + const struct cpumask **affinity) { + int ret; #ifdef CONFIG_SPARC /* sparc does not have irqs represented as IORESOURCE_IRQ resources */ if (!dev || num >= dev->archdata.num_irqs) - return -ENXIO; - return dev->archdata.irqs[num]; + goto out_not_found; + ret = dev->archdata.irqs[num]; + goto out; #else - struct resource *r = platform_get_resource(dev, IORESOURCE_IRQ, num); + struct fwnode_handle *fwnode = dev_fwnode(&dev->dev); + struct resource *r; + + if (is_of_node(fwnode)) { + ret = of_irq_get(to_of_node(fwnode), num); + if (ret > 0 || ret == -EPROBE_DEFER) + goto out; + } + + r = platform_get_resource(dev, IORESOURCE_IRQ, num); + if (is_acpi_device_node(fwnode)) { + if (r && r->flags & IORESOURCE_DISABLED) { + ret = acpi_irq_get(ACPI_HANDLE_FWNODE(fwnode), num, r); + if (ret) + goto out; + } + } + + /* + * The resources may pass trigger flags to the irqs that need + * to be set up. It so happens that the trigger flags for + * IORESOURCE_BITS correspond 1-to-1 to the IRQF_TRIGGER* + * settings. + */ + if (r && r->flags & IORESOURCE_BITS) { + struct irq_data *irqd; + + irqd = irq_get_irq_data(r->start); + if (!irqd) + goto out_not_found; + irqd_set_trigger_type(irqd, r->flags & IORESOURCE_BITS); + } + + if (r) { + ret = r->start; + goto out; + } + + /* + * For the index 0 interrupt, allow falling back to GpioInt + * resources. While a device could have both Interrupt and GpioInt + * resources, making this fallback ambiguous, in many common cases + * the device will only expose one IRQ, and this fallback + * allows a common code path across either kind of resource. + */ + if (num == 0 && is_acpi_device_node(fwnode)) { + ret = acpi_dev_gpio_irq_get(to_acpi_device_node(fwnode), num); + /* Our callers expect -ENXIO for missing IRQs. */ + if (ret >= 0 || ret == -EPROBE_DEFER) + goto out; + } - return r ? r->start : -ENXIO; #endif +out_not_found: + ret = -ENXIO; +out: + if (WARN(!ret, "0 is an invalid IRQ number\n")) + return -EINVAL; + + if (ret > 0 && affinity) + *affinity = get_irq_affinity(dev, num); + + return ret; +} +EXPORT_SYMBOL_GPL(platform_get_irq_affinity); + +/** + * platform_get_irq_optional - get an optional interrupt for a device + * @dev: platform device + * @num: interrupt number index + * + * Gets an interrupt for a platform device. Device drivers should check the + * return value for errors so as to not pass a negative integer value to + * the request_irq() APIs. This is the same as platform_get_irq(), except + * that it does not print an error message if an interrupt can not be + * obtained. + * + * For example:: + * + * int irq = platform_get_irq_optional(pdev, 0); + * if (irq < 0) + * return irq; + * + * Return: non-zero interrupt number on success, negative error number on failure. + */ +int platform_get_irq_optional(struct platform_device *dev, unsigned int num) +{ + return platform_get_irq_affinity(dev, num, NULL); +} +EXPORT_SYMBOL_GPL(platform_get_irq_optional); + +/** + * platform_get_irq - get an IRQ for a device + * @dev: platform device + * @num: IRQ number index + * + * Gets an IRQ for a platform device and prints an error message if finding the + * IRQ fails. Device drivers should check the return value for errors so as to + * not pass a negative integer value to the request_irq() APIs. + * + * For example:: + * + * int irq = platform_get_irq(pdev, 0); + * if (irq < 0) + * return irq; + * + * Return: non-zero IRQ number on success, negative error number on failure. + */ +int platform_get_irq(struct platform_device *dev, unsigned int num) +{ + int ret; + + ret = platform_get_irq_optional(dev, num); + if (ret < 0) + return dev_err_probe(&dev->dev, ret, + "IRQ index %u not found\n", num); + + return ret; } EXPORT_SYMBOL_GPL(platform_get_irq); /** + * platform_irq_count - Count the number of IRQs a platform device uses + * @dev: platform device + * + * Return: Number of IRQs a platform device uses or EPROBE_DEFER + */ +int platform_irq_count(struct platform_device *dev) +{ + int ret, nr = 0; + + while ((ret = platform_get_irq_optional(dev, nr)) >= 0) + nr++; + + if (ret == -EPROBE_DEFER) + return ret; + + return nr; +} +EXPORT_SYMBOL_GPL(platform_irq_count); + +struct irq_affinity_devres { + unsigned int count; + unsigned int irq[] __counted_by(count); +}; + +static void platform_disable_acpi_irq(struct platform_device *pdev, int index) +{ + struct resource *r; + + r = platform_get_resource(pdev, IORESOURCE_IRQ, index); + if (r) + irqresource_disabled(r, 0); +} + +static void devm_platform_get_irqs_affinity_release(struct device *dev, + void *res) +{ + struct irq_affinity_devres *ptr = res; + int i; + + for (i = 0; i < ptr->count; i++) { + irq_dispose_mapping(ptr->irq[i]); + + if (is_acpi_device_node(dev_fwnode(dev))) + platform_disable_acpi_irq(to_platform_device(dev), i); + } +} + +/** + * devm_platform_get_irqs_affinity - devm method to get a set of IRQs for a + * device using an interrupt affinity descriptor + * @dev: platform device pointer + * @affd: affinity descriptor + * @minvec: minimum count of interrupt vectors + * @maxvec: maximum count of interrupt vectors + * @irqs: pointer holder for IRQ numbers + * + * Gets a set of IRQs for a platform device, and updates IRQ afffinty according + * to the passed affinity descriptor + * + * Return: Number of vectors on success, negative error number on failure. + */ +int devm_platform_get_irqs_affinity(struct platform_device *dev, + struct irq_affinity *affd, + unsigned int minvec, + unsigned int maxvec, + int **irqs) +{ + struct irq_affinity_devres *ptr; + struct irq_affinity_desc *desc; + size_t size; + int i, ret, nvec; + + if (!affd) + return -EPERM; + + if (maxvec < minvec) + return -ERANGE; + + nvec = platform_irq_count(dev); + if (nvec < 0) + return nvec; + + if (nvec < minvec) + return -ENOSPC; + + nvec = irq_calc_affinity_vectors(minvec, nvec, affd); + if (nvec < minvec) + return -ENOSPC; + + if (nvec > maxvec) + nvec = maxvec; + + size = sizeof(*ptr) + sizeof(unsigned int) * nvec; + ptr = devres_alloc(devm_platform_get_irqs_affinity_release, size, + GFP_KERNEL); + if (!ptr) + return -ENOMEM; + + ptr->count = nvec; + + for (i = 0; i < nvec; i++) { + int irq = platform_get_irq(dev, i); + if (irq < 0) { + ret = irq; + goto err_free_devres; + } + ptr->irq[i] = irq; + } + + desc = irq_create_affinity_masks(nvec, affd); + if (!desc) { + ret = -ENOMEM; + goto err_free_devres; + } + + for (i = 0; i < nvec; i++) { + ret = irq_update_affinity_desc(ptr->irq[i], &desc[i]); + if (ret) { + dev_err(&dev->dev, "failed to update irq%d affinity descriptor (%d)\n", + ptr->irq[i], ret); + goto err_free_desc; + } + } + + devres_add(&dev->dev, ptr); + + kfree(desc); + + *irqs = ptr->irq; + + return nvec; + +err_free_desc: + kfree(desc); +err_free_devres: + devres_free(ptr); + return ret; +} +EXPORT_SYMBOL_GPL(devm_platform_get_irqs_affinity); + +/** * platform_get_resource_byname - get a resource for a device by name * @dev: platform device * @type: resource type @@ -104,7 +462,7 @@ struct resource *platform_get_resource_byname(struct platform_device *dev, unsigned int type, const char *name) { - int i; + u32 i; for (i = 0; i < dev->num_resources; i++) { struct resource *r = &dev->resource[i]; @@ -119,24 +477,70 @@ struct resource *platform_get_resource_byname(struct platform_device *dev, } EXPORT_SYMBOL_GPL(platform_get_resource_byname); +static int __platform_get_irq_byname(struct platform_device *dev, + const char *name) +{ + struct resource *r; + int ret; + + ret = fwnode_irq_get_byname(dev_fwnode(&dev->dev), name); + if (ret > 0 || ret == -EPROBE_DEFER) + return ret; + + r = platform_get_resource_byname(dev, IORESOURCE_IRQ, name); + if (r) { + if (WARN(!r->start, "0 is an invalid IRQ number\n")) + return -EINVAL; + return r->start; + } + + return -ENXIO; +} + /** * platform_get_irq_byname - get an IRQ for a device by name * @dev: platform device * @name: IRQ name + * + * Get an IRQ like platform_get_irq(), but then by name rather then by index. + * + * Return: non-zero IRQ number on success, negative error number on failure. */ int platform_get_irq_byname(struct platform_device *dev, const char *name) { - struct resource *r = platform_get_resource_byname(dev, IORESOURCE_IRQ, - name); + int ret; - return r ? r->start : -ENXIO; + ret = __platform_get_irq_byname(dev, name); + if (ret < 0) + return dev_err_probe(&dev->dev, ret, "IRQ %s not found\n", + name); + return ret; } EXPORT_SYMBOL_GPL(platform_get_irq_byname); /** + * platform_get_irq_byname_optional - get an optional IRQ for a device by name + * @dev: platform device + * @name: IRQ name + * + * Get an optional IRQ by name like platform_get_irq_byname(). Except that it + * does not print an error message if an IRQ can not be obtained. + * + * Return: non-zero IRQ number on success, negative error number on failure. + */ +int platform_get_irq_byname_optional(struct platform_device *dev, + const char *name) +{ + return __platform_get_irq_byname(dev, name); +} +EXPORT_SYMBOL_GPL(platform_get_irq_byname_optional); + +/** * platform_add_devices - add a numbers of platform devices * @devs: array of platform devices to add * @num: number of platform devices in array + * + * Return: 0 on success, negative error number on failure. */ int platform_add_devices(struct platform_device **devs, int num) { @@ -157,7 +561,23 @@ EXPORT_SYMBOL_GPL(platform_add_devices); struct platform_object { struct platform_device pdev; - char name[1]; + char name[]; +}; + +/* + * Set up default DMA mask for platform devices if the they weren't + * previously set by the architecture / DT. + */ +static void setup_pdev_dma_masks(struct platform_device *pdev) +{ + pdev->dev.dma_parms = &pdev->dma_parms; + + if (!pdev->dev.coherent_dma_mask) + pdev->dev.coherent_dma_mask = DMA_BIT_MASK(32); + if (!pdev->dev.dma_mask) { + pdev->platform_dma_mask = DMA_BIT_MASK(32); + pdev->dev.dma_mask = &pdev->platform_dma_mask; + } }; /** @@ -169,7 +589,7 @@ struct platform_object { */ void platform_device_put(struct platform_device *pdev) { - if (pdev) + if (!IS_ERR_OR_NULL(pdev)) put_device(&pdev->dev); } EXPORT_SYMBOL_GPL(platform_device_put); @@ -179,10 +599,11 @@ static void platform_device_release(struct device *dev) struct platform_object *pa = container_of(dev, struct platform_object, pdev.dev); - of_device_node_put(&pa->pdev.dev); + of_node_put(pa->pdev.dev.of_node); kfree(pa->pdev.dev.platform_data); kfree(pa->pdev.mfd_cell); kfree(pa->pdev.resource); + kfree(pa->pdev.driver_override); kfree(pa); } @@ -198,14 +619,14 @@ struct platform_device *platform_device_alloc(const char *name, int id) { struct platform_object *pa; - pa = kzalloc(sizeof(struct platform_object) + strlen(name), GFP_KERNEL); + pa = kzalloc(sizeof(*pa) + strlen(name) + 1, GFP_KERNEL); if (pa) { strcpy(pa->name, name); pa->pdev.name = pa->name; pa->pdev.id = id; device_initialize(&pa->pdev.dev); pa->pdev.dev.release = platform_device_release; - arch_setup_pdev_archdata(&pa->pdev); + setup_pdev_dma_masks(&pa->pdev); } return pa ? &pa->pdev : NULL; @@ -228,7 +649,7 @@ int platform_device_add_resources(struct platform_device *pdev, struct resource *r = NULL; if (res) { - r = kmemdup(res, sizeof(struct resource) * num, GFP_KERNEL); + r = kmemdup_array(res, num, sizeof(*r), GFP_KERNEL); if (!r) return -ENOMEM; } @@ -276,22 +697,21 @@ EXPORT_SYMBOL_GPL(platform_device_add_data); */ int platform_device_add(struct platform_device *pdev) { - int i, ret; - - if (!pdev) - return -EINVAL; + struct device *dev = &pdev->dev; + u32 i; + int ret; - if (!pdev->dev.parent) - pdev->dev.parent = &platform_bus; + if (!dev->parent) + dev->parent = &platform_bus; - pdev->dev.bus = &platform_bus_type; + dev->bus = &platform_bus_type; switch (pdev->id) { default: - dev_set_name(&pdev->dev, "%s.%d", pdev->name, pdev->id); + dev_set_name(dev, "%s.%d", pdev->name, pdev->id); break; case PLATFORM_DEVID_NONE: - dev_set_name(&pdev->dev, "%s", pdev->name); + dev_set_name(dev, "%s", pdev->name); break; case PLATFORM_DEVID_AUTO: /* @@ -299,12 +719,12 @@ int platform_device_add(struct platform_device *pdev) * that we remember it must be freed, and we append a suffix * to avoid namespace collision with explicit IDs. */ - ret = ida_simple_get(&platform_devid_ida, 0, 0, GFP_KERNEL); + ret = ida_alloc(&platform_devid_ida, GFP_KERNEL); if (ret < 0) - goto err_out; + return ret; pdev->id = ret; pdev->id_auto = true; - dev_set_name(&pdev->dev, "%s.%d.auto", pdev->name, pdev->id); + dev_set_name(dev, "%s.%d.auto", pdev->name, pdev->id); break; } @@ -312,7 +732,7 @@ int platform_device_add(struct platform_device *pdev) struct resource *p, *r = &pdev->resource[i]; if (r->name == NULL) - r->name = dev_name(&pdev->dev); + r->name = dev_name(dev); p = r->parent; if (!p) { @@ -322,35 +742,36 @@ int platform_device_add(struct platform_device *pdev) p = &ioport_resource; } - if (p && insert_resource(p, r)) { - dev_err(&pdev->dev, "failed to claim resource %d\n", i); - ret = -EBUSY; - goto failed; + if (p) { + ret = insert_resource(p, r); + if (ret) { + dev_err(dev, "failed to claim resource %d: %pR\n", i, r); + goto failed; + } } } - pr_debug("Registering platform device '%s'. Parent at %s\n", - dev_name(&pdev->dev), dev_name(pdev->dev.parent)); + pr_debug("Registering platform device '%s'. Parent at %s\n", dev_name(dev), + dev_name(dev->parent)); - ret = device_add(&pdev->dev); - if (ret == 0) - return ret; + ret = device_add(dev); + if (ret) + goto failed; + + return 0; failed: if (pdev->id_auto) { - ida_simple_remove(&platform_devid_ida, pdev->id); + ida_free(&platform_devid_ida, pdev->id); pdev->id = PLATFORM_DEVID_AUTO; } - while (--i >= 0) { + while (i--) { struct resource *r = &pdev->resource[i]; - unsigned long type = resource_type(r); - - if (type == IORESOURCE_MEM || type == IORESOURCE_IO) + if (r->parent) release_resource(r); } - err_out: return ret; } EXPORT_SYMBOL_GPL(platform_device_add); @@ -365,21 +786,19 @@ EXPORT_SYMBOL_GPL(platform_device_add); */ void platform_device_del(struct platform_device *pdev) { - int i; + u32 i; - if (pdev) { + if (!IS_ERR_OR_NULL(pdev)) { device_del(&pdev->dev); if (pdev->id_auto) { - ida_simple_remove(&platform_devid_ida, pdev->id); + ida_free(&platform_devid_ida, pdev->id); pdev->id = PLATFORM_DEVID_AUTO; } for (i = 0; i < pdev->num_resources; i++) { struct resource *r = &pdev->resource[i]; - unsigned long type = resource_type(r); - - if (type == IORESOURCE_MEM || type == IORESOURCE_IO) + if (r->parent) release_resource(r); } } @@ -389,11 +808,15 @@ EXPORT_SYMBOL_GPL(platform_device_del); /** * platform_device_register - add a platform-level device * @pdev: platform device we're adding + * + * NOTE: _Never_ directly free @pdev after calling this function, even if it + * returned an error! Always use platform_device_put() to give up the + * reference initialised in this function instead. */ int platform_device_register(struct platform_device *pdev) { device_initialize(&pdev->dev); - arch_setup_pdev_archdata(pdev); + setup_pdev_dma_masks(pdev); return platform_device_add(pdev); } EXPORT_SYMBOL_GPL(platform_device_register); @@ -424,29 +847,21 @@ EXPORT_SYMBOL_GPL(platform_device_unregister); struct platform_device *platform_device_register_full( const struct platform_device_info *pdevinfo) { - int ret = -ENOMEM; + int ret; struct platform_device *pdev; pdev = platform_device_alloc(pdevinfo->name, pdevinfo->id); if (!pdev) - goto err_alloc; + return ERR_PTR(-ENOMEM); pdev->dev.parent = pdevinfo->parent; - ACPI_HANDLE_SET(&pdev->dev, pdevinfo->acpi_node.handle); + pdev->dev.fwnode = pdevinfo->fwnode; + pdev->dev.of_node = of_node_get(to_of_node(pdev->dev.fwnode)); + pdev->dev.of_node_reused = pdevinfo->of_node_reused; if (pdevinfo->dma_mask) { - /* - * This memory isn't freed when the device is put, - * I don't have a nice idea for that though. Conceptually - * dma_mask in struct device should not be a pointer. - * See http://thread.gmane.org/gmane.linux.kernel.pci/9081 - */ - pdev->dev.dma_mask = - kmalloc(sizeof(*pdev->dev.dma_mask), GFP_KERNEL); - if (!pdev->dev.dma_mask) - goto err; - - *pdev->dev.dma_mask = pdevinfo->dma_mask; + pdev->platform_dma_mask = pdevinfo->dma_mask; + pdev->dev.dma_mask = &pdev->platform_dma_mask; pdev->dev.coherent_dma_mask = pdevinfo->dma_mask; } @@ -460,13 +875,17 @@ struct platform_device *platform_device_register_full( if (ret) goto err; + if (pdevinfo->properties) { + ret = device_create_managed_software_node(&pdev->dev, + pdevinfo->properties, NULL); + if (ret) + goto err; + } + ret = platform_device_add(pdev); if (ret) { err: - ACPI_HANDLE_SET(&pdev->dev, NULL); - kfree(pdev->dev.dma_mask); - -err_alloc: + ACPI_COMPANION_SET(&pdev->dev, NULL); platform_device_put(pdev); return ERR_PTR(ret); } @@ -475,65 +894,16 @@ err_alloc: } EXPORT_SYMBOL_GPL(platform_device_register_full); -static int platform_drv_probe(struct device *_dev) -{ - struct platform_driver *drv = to_platform_driver(_dev->driver); - struct platform_device *dev = to_platform_device(_dev); - int ret; - - if (ACPI_HANDLE(_dev)) - acpi_dev_pm_attach(_dev, true); - - ret = drv->probe(dev); - if (ret && ACPI_HANDLE(_dev)) - acpi_dev_pm_detach(_dev, true); - - return ret; -} - -static int platform_drv_probe_fail(struct device *_dev) -{ - return -ENXIO; -} - -static int platform_drv_remove(struct device *_dev) -{ - struct platform_driver *drv = to_platform_driver(_dev->driver); - struct platform_device *dev = to_platform_device(_dev); - int ret; - - ret = drv->remove(dev); - if (ACPI_HANDLE(_dev)) - acpi_dev_pm_detach(_dev, true); - - return ret; -} - -static void platform_drv_shutdown(struct device *_dev) -{ - struct platform_driver *drv = to_platform_driver(_dev->driver); - struct platform_device *dev = to_platform_device(_dev); - - drv->shutdown(dev); - if (ACPI_HANDLE(_dev)) - acpi_dev_pm_detach(_dev, true); -} - /** * __platform_driver_register - register a driver for platform-level devices * @drv: platform driver structure + * @owner: owning module/driver */ int __platform_driver_register(struct platform_driver *drv, struct module *owner) { drv->driver.owner = owner; drv->driver.bus = &platform_bus_type; - if (drv->probe) - drv->driver.probe = platform_drv_probe; - if (drv->remove) - drv->driver.remove = platform_drv_remove; - if (drv->shutdown) - drv->driver.shutdown = platform_drv_shutdown; return driver_register(&drv->driver); } @@ -549,11 +919,23 @@ void platform_driver_unregister(struct platform_driver *drv) } EXPORT_SYMBOL_GPL(platform_driver_unregister); +static int platform_probe_fail(struct platform_device *pdev) +{ + return -ENXIO; +} + +static int is_bound_to_driver(struct device *dev, void *driver) +{ + if (dev->driver == driver) + return 1; + return 0; +} + /** - * platform_driver_probe - register driver for non-hotpluggable device + * __platform_driver_probe - register driver for non-hotpluggable device * @drv: platform driver structure - * @probe: the driver probe routine, probably from an __init section, - * must not return -EPROBE_DEFER. + * @probe: the driver probe routine, probably from an __init section + * @module: module which will be the owner of the driver * * Use this instead of platform_driver_register() when you know the device * is not hotpluggable and has already been registered, and you want to @@ -564,67 +946,84 @@ EXPORT_SYMBOL_GPL(platform_driver_unregister); * into system-on-chip processors, where the controller devices have been * configured as part of board setup. * - * This is incompatible with deferred probing so probe() must not - * return -EPROBE_DEFER. + * Note that this is incompatible with deferred probing. * * Returns zero if the driver registered and bound to a device, else returns * a negative error code and with the driver not registered. */ -int __init_or_module platform_driver_probe(struct platform_driver *drv, - int (*probe)(struct platform_device *)) +int __init_or_module __platform_driver_probe(struct platform_driver *drv, + int (*probe)(struct platform_device *), struct module *module) { - int retval, code; + int retval; + + if (drv->driver.probe_type == PROBE_PREFER_ASYNCHRONOUS) { + pr_err("%s: drivers registered with %s can not be probed asynchronously\n", + drv->driver.name, __func__); + return -EINVAL; + } + + /* + * We have to run our probes synchronously because we check if + * we find any devices to bind to and exit with error if there + * are any. + */ + drv->driver.probe_type = PROBE_FORCE_SYNCHRONOUS; + + /* + * Prevent driver from requesting probe deferral to avoid further + * futile probe attempts. + */ + drv->prevent_deferred_probe = true; /* make sure driver won't have bind/unbind attributes */ drv->driver.suppress_bind_attrs = true; /* temporary section violation during probe() */ drv->probe = probe; - retval = code = platform_driver_register(drv); + retval = __platform_driver_register(drv, module); + if (retval) + return retval; - /* - * Fixup that section violation, being paranoid about code scanning - * the list of drivers in order to probe new devices. Check to see - * if the probe was successful, and make sure any forced probes of - * new devices fail. + /* Force all new probes of this driver to fail */ + drv->probe = platform_probe_fail; + + /* Walk all platform devices and see if any actually bound to this driver. + * If not, return an error as the device should have done so by now. */ - spin_lock(&drv->driver.bus->p->klist_drivers.k_lock); - drv->probe = NULL; - if (code == 0 && list_empty(&drv->driver.p->klist_devices.k_list)) + if (!bus_for_each_dev(&platform_bus_type, NULL, &drv->driver, is_bound_to_driver)) { retval = -ENODEV; - drv->driver.probe = platform_drv_probe_fail; - spin_unlock(&drv->driver.bus->p->klist_drivers.k_lock); - - if (code != retval) platform_driver_unregister(drv); + } + return retval; } -EXPORT_SYMBOL_GPL(platform_driver_probe); +EXPORT_SYMBOL_GPL(__platform_driver_probe); /** - * platform_create_bundle - register driver and create corresponding device + * __platform_create_bundle - register driver and create corresponding device * @driver: platform driver structure * @probe: the driver probe routine, probably from an __init section * @res: set of resources that needs to be allocated for the device * @n_res: number of resources * @data: platform specific data for this platform device * @size: size of platform specific data + * @module: module which will be the owner of the driver * * Use this in legacy-style modules that probe hardware directly and * register a single platform device and corresponding platform driver. * * Returns &struct platform_device pointer on success, or ERR_PTR() on error. */ -struct platform_device * __init_or_module platform_create_bundle( +struct platform_device * __init_or_module __platform_create_bundle( struct platform_driver *driver, int (*probe)(struct platform_device *), struct resource *res, unsigned int n_res, - const void *data, size_t size) + const void *data, size_t size, struct module *module) { struct platform_device *pdev; int error; - pdev = platform_device_alloc(driver->driver.name, -1); + pdev = platform_device_alloc(driver->driver.name, PLATFORM_DEVID_NONE); if (!pdev) { error = -ENOMEM; goto err_out; @@ -642,7 +1041,7 @@ struct platform_device * __init_or_module platform_create_bundle( if (error) goto err_pdev_put; - error = platform_driver_probe(driver, probe); + error = __platform_driver_probe(driver, probe, module); if (error) goto err_pdev_del; @@ -655,42 +1054,68 @@ err_pdev_put: err_out: return ERR_PTR(error); } -EXPORT_SYMBOL_GPL(platform_create_bundle); +EXPORT_SYMBOL_GPL(__platform_create_bundle); -/* modalias support enables more hands-off userspace setup: - * (a) environment variable lets new-style hotplug events work once system is - * fully running: "modprobe $MODALIAS" - * (b) sysfs attribute lets new-style coldplug recover from hotplug events - * mishandled before system is fully running: "modprobe $(cat modalias)" +/** + * __platform_register_drivers - register an array of platform drivers + * @drivers: an array of drivers to register + * @count: the number of drivers to register + * @owner: module owning the drivers + * + * Registers platform drivers specified by an array. On failure to register a + * driver, all previously registered drivers will be unregistered. Callers of + * this API should use platform_unregister_drivers() to unregister drivers in + * the reverse order. + * + * Returns: 0 on success or a negative error code on failure. */ -static ssize_t modalias_show(struct device *dev, struct device_attribute *a, - char *buf) +int __platform_register_drivers(struct platform_driver * const *drivers, + unsigned int count, struct module *owner) { - struct platform_device *pdev = to_platform_device(dev); - int len = snprintf(buf, PAGE_SIZE, "platform:%s\n", pdev->name); + unsigned int i; + int err; - return (len >= PAGE_SIZE) ? (PAGE_SIZE - 1) : len; -} + for (i = 0; i < count; i++) { + pr_debug("registering platform driver %ps\n", drivers[i]); -static struct device_attribute platform_dev_attrs[] = { - __ATTR_RO(modalias), - __ATTR_NULL, -}; + err = __platform_driver_register(drivers[i], owner); + if (err < 0) { + pr_err("failed to register platform driver %ps: %d\n", + drivers[i], err); + goto error; + } + } -static int platform_uevent(struct device *dev, struct kobj_uevent_env *env) -{ - struct platform_device *pdev = to_platform_device(dev); - int rc; + return 0; - /* Some devices have extra OF data and an OF-style MODALIAS */ - rc = of_device_uevent_modalias(dev, env); - if (rc != -ENODEV) - return rc; +error: + while (i--) { + pr_debug("unregistering platform driver %ps\n", drivers[i]); + platform_driver_unregister(drivers[i]); + } - add_uevent_var(env, "MODALIAS=%s%s", PLATFORM_MODULE_PREFIX, - pdev->name); - return 0; + return err; +} +EXPORT_SYMBOL_GPL(__platform_register_drivers); + +/** + * platform_unregister_drivers - unregister an array of platform drivers + * @drivers: an array of drivers to unregister + * @count: the number of drivers to unregister + * + * Unregisters platform drivers specified by an array. This is typically used + * to complement an earlier call to platform_register_drivers(). Drivers are + * unregistered in the reverse order in which they were registered. + */ +void platform_unregister_drivers(struct platform_driver * const *drivers, + unsigned int count) +{ + while (count--) { + pr_debug("unregistering platform driver %ps\n", drivers[count]); + platform_driver_unregister(drivers[count]); + } } +EXPORT_SYMBOL_GPL(platform_unregister_drivers); static const struct platform_device_id *platform_match_id( const struct platform_device_id *id, @@ -706,40 +1131,6 @@ static const struct platform_device_id *platform_match_id( return NULL; } -/** - * platform_match - bind platform device to platform driver. - * @dev: device. - * @drv: driver. - * - * Platform device IDs are assumed to be encoded like this: - * "<name><instance>", where <name> is a short description of the type of - * device, like "pci" or "floppy", and <instance> is the enumerated - * instance of the device, like '0' or '42'. Driver IDs are simply - * "<name>". So, extract the <name> from the platform_device structure, - * and compare it against the name of the driver. Return whether they match - * or not. - */ -static int platform_match(struct device *dev, struct device_driver *drv) -{ - struct platform_device *pdev = to_platform_device(dev); - struct platform_driver *pdrv = to_platform_driver(drv); - - /* Attempt an OF style match first */ - if (of_driver_match_device(dev, drv)) - return 1; - - /* Then try ACPI style match */ - if (acpi_driver_match_device(dev, drv)) - return 1; - - /* Then try to match against the id table */ - if (pdrv->id_table) - return platform_match_id(pdrv->id_table, pdev) != NULL; - - /* fall-back to driver name match */ - return (strcmp(pdev->name, drv->name) == 0); -} - #ifdef CONFIG_PM_SLEEP static int platform_legacy_suspend(struct device *dev, pm_message_t mesg) @@ -772,7 +1163,7 @@ static int platform_legacy_resume(struct device *dev) int platform_pm_suspend(struct device *dev) { - struct device_driver *drv = dev->driver; + const struct device_driver *drv = dev->driver; int ret = 0; if (!drv) @@ -790,7 +1181,7 @@ int platform_pm_suspend(struct device *dev) int platform_pm_resume(struct device *dev) { - struct device_driver *drv = dev->driver; + const struct device_driver *drv = dev->driver; int ret = 0; if (!drv) @@ -812,7 +1203,7 @@ int platform_pm_resume(struct device *dev) int platform_pm_freeze(struct device *dev) { - struct device_driver *drv = dev->driver; + const struct device_driver *drv = dev->driver; int ret = 0; if (!drv) @@ -830,7 +1221,7 @@ int platform_pm_freeze(struct device *dev) int platform_pm_thaw(struct device *dev) { - struct device_driver *drv = dev->driver; + const struct device_driver *drv = dev->driver; int ret = 0; if (!drv) @@ -848,7 +1239,7 @@ int platform_pm_thaw(struct device *dev) int platform_pm_poweroff(struct device *dev) { - struct device_driver *drv = dev->driver; + const struct device_driver *drv = dev->driver; int ret = 0; if (!drv) @@ -866,7 +1257,7 @@ int platform_pm_poweroff(struct device *dev) int platform_pm_restore(struct device *dev) { - struct device_driver *drv = dev->driver; + const struct device_driver *drv = dev->driver; int ret = 0; if (!drv) @@ -884,340 +1275,293 @@ int platform_pm_restore(struct device *dev) #endif /* CONFIG_HIBERNATE_CALLBACKS */ -static const struct dev_pm_ops platform_dev_pm_ops = { - .runtime_suspend = pm_generic_runtime_suspend, - .runtime_resume = pm_generic_runtime_resume, - USE_PLATFORM_PM_SLEEP_OPS -}; - -struct bus_type platform_bus_type = { - .name = "platform", - .dev_attrs = platform_dev_attrs, - .match = platform_match, - .uevent = platform_uevent, - .pm = &platform_dev_pm_ops, -}; -EXPORT_SYMBOL_GPL(platform_bus_type); - -int __init platform_bus_init(void) +/* modalias support enables more hands-off userspace setup: + * (a) environment variable lets new-style hotplug events work once system is + * fully running: "modprobe $MODALIAS" + * (b) sysfs attribute lets new-style coldplug recover from hotplug events + * mishandled before system is fully running: "modprobe $(cat modalias)" + */ +static ssize_t modalias_show(struct device *dev, + struct device_attribute *attr, char *buf) { - int error; + struct platform_device *pdev = to_platform_device(dev); + int len; - early_platform_cleanup(); + len = of_device_modalias(dev, buf, PAGE_SIZE); + if (len != -ENODEV) + return len; - error = device_register(&platform_bus); - if (error) - return error; - error = bus_register(&platform_bus_type); - if (error) - device_unregister(&platform_bus); - return error; + len = acpi_device_modalias(dev, buf, PAGE_SIZE - 1); + if (len != -ENODEV) + return len; + + return sysfs_emit(buf, "platform:%s\n", pdev->name); } +static DEVICE_ATTR_RO(modalias); -#ifndef ARCH_HAS_DMA_GET_REQUIRED_MASK -u64 dma_get_required_mask(struct device *dev) +static ssize_t numa_node_show(struct device *dev, + struct device_attribute *attr, char *buf) { - u32 low_totalram = ((max_pfn - 1) << PAGE_SHIFT); - u32 high_totalram = ((max_pfn - 1) >> (32 - PAGE_SHIFT)); - u64 mask; - - if (!high_totalram) { - /* convert to mask just covering totalram */ - low_totalram = (1 << (fls(low_totalram) - 1)); - low_totalram += low_totalram - 1; - mask = low_totalram; - } else { - high_totalram = (1 << (fls(high_totalram) - 1)); - high_totalram += high_totalram - 1; - mask = (((u64)high_totalram) << 32) + 0xffffffff; - } - return mask; + return sysfs_emit(buf, "%d\n", dev_to_node(dev)); } -EXPORT_SYMBOL_GPL(dma_get_required_mask); -#endif +static DEVICE_ATTR_RO(numa_node); -static __initdata LIST_HEAD(early_platform_driver_list); -static __initdata LIST_HEAD(early_platform_device_list); - -/** - * early_platform_driver_register - register early platform driver - * @epdrv: early_platform driver structure - * @buf: string passed from early_param() - * - * Helper function for early_platform_init() / early_platform_init_buffer() - */ -int __init early_platform_driver_register(struct early_platform_driver *epdrv, - char *buf) +static ssize_t driver_override_show(struct device *dev, + struct device_attribute *attr, char *buf) { - char *tmp; - int n; + struct platform_device *pdev = to_platform_device(dev); + ssize_t len; - /* Simply add the driver to the end of the global list. - * Drivers will by default be put on the list in compiled-in order. - */ - if (!epdrv->list.next) { - INIT_LIST_HEAD(&epdrv->list); - list_add_tail(&epdrv->list, &early_platform_driver_list); - } + device_lock(dev); + len = sysfs_emit(buf, "%s\n", pdev->driver_override); + device_unlock(dev); - /* If the user has specified device then make sure the driver - * gets prioritized. The driver of the last device specified on - * command line will be put first on the list. - */ - n = strlen(epdrv->pdrv->driver.name); - if (buf && !strncmp(buf, epdrv->pdrv->driver.name, n)) { - list_move(&epdrv->list, &early_platform_driver_list); - - /* Allow passing parameters after device name */ - if (buf[n] == '\0' || buf[n] == ',') - epdrv->requested_id = -1; - else { - epdrv->requested_id = simple_strtoul(&buf[n + 1], - &tmp, 10); - - if (buf[n] != '.' || (tmp == &buf[n + 1])) { - epdrv->requested_id = EARLY_PLATFORM_ID_ERROR; - n = 0; - } else - n += strcspn(&buf[n + 1], ",") + 1; - } + return len; +} - if (buf[n] == ',') - n++; +static ssize_t driver_override_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct platform_device *pdev = to_platform_device(dev); + int ret; - if (epdrv->bufsize) { - memcpy(epdrv->buffer, &buf[n], - min_t(int, epdrv->bufsize, strlen(&buf[n]) + 1)); - epdrv->buffer[epdrv->bufsize - 1] = '\0'; - } - } + ret = driver_set_override(dev, &pdev->driver_override, buf, count); + if (ret) + return ret; - return 0; + return count; } +static DEVICE_ATTR_RW(driver_override); -/** - * early_platform_add_devices - adds a number of early platform devices - * @devs: array of early platform devices to add - * @num: number of early platform devices in array - * - * Used by early architecture code to register early platform devices and - * their platform data. - */ -void __init early_platform_add_devices(struct platform_device **devs, int num) +static struct attribute *platform_dev_attrs[] = { + &dev_attr_modalias.attr, + &dev_attr_numa_node.attr, + &dev_attr_driver_override.attr, + NULL, +}; + +static umode_t platform_dev_attrs_visible(struct kobject *kobj, struct attribute *a, + int n) { - struct device *dev; - int i; + struct device *dev = container_of(kobj, typeof(*dev), kobj); - /* simply add the devices to list */ - for (i = 0; i < num; i++) { - dev = &devs[i]->dev; + if (a == &dev_attr_numa_node.attr && + dev_to_node(dev) == NUMA_NO_NODE) + return 0; - if (!dev->devres_head.next) { - pm_runtime_early_init(dev); - INIT_LIST_HEAD(&dev->devres_head); - list_add_tail(&dev->devres_head, - &early_platform_device_list); - } - } + return a->mode; } +static const struct attribute_group platform_dev_group = { + .attrs = platform_dev_attrs, + .is_visible = platform_dev_attrs_visible, +}; +__ATTRIBUTE_GROUPS(platform_dev); + + /** - * early_platform_driver_register_all - register early platform drivers - * @class_str: string to identify early platform driver class + * platform_match - bind platform device to platform driver. + * @dev: device. + * @drv: driver. * - * Used by architecture code to register all early platform drivers - * for a certain class. If omitted then only early platform drivers - * with matching kernel command line class parameters will be registered. + * Platform device IDs are assumed to be encoded like this: + * "<name><instance>", where <name> is a short description of the type of + * device, like "pci" or "floppy", and <instance> is the enumerated + * instance of the device, like '0' or '42'. Driver IDs are simply + * "<name>". So, extract the <name> from the platform_device structure, + * and compare it against the name of the driver. Return whether they match + * or not. */ -void __init early_platform_driver_register_all(char *class_str) +static int platform_match(struct device *dev, const struct device_driver *drv) { - /* The "class_str" parameter may or may not be present on the kernel - * command line. If it is present then there may be more than one - * matching parameter. - * - * Since we register our early platform drivers using early_param() - * we need to make sure that they also get registered in the case - * when the parameter is missing from the kernel command line. - * - * We use parse_early_options() to make sure the early_param() gets - * called at least once. The early_param() may be called more than - * once since the name of the preferred device may be specified on - * the kernel command line. early_platform_driver_register() handles - * this case for us. - */ - parse_early_options(class_str); -} + struct platform_device *pdev = to_platform_device(dev); + struct platform_driver *pdrv = to_platform_driver(drv); -/** - * early_platform_match - find early platform device matching driver - * @epdrv: early platform driver structure - * @id: id to match against - */ -static __init struct platform_device * -early_platform_match(struct early_platform_driver *epdrv, int id) -{ - struct platform_device *pd; + /* When driver_override is set, only bind to the matching driver */ + if (pdev->driver_override) + return !strcmp(pdev->driver_override, drv->name); - list_for_each_entry(pd, &early_platform_device_list, dev.devres_head) - if (platform_match(&pd->dev, &epdrv->pdrv->driver)) - if (pd->id == id) - return pd; + /* Attempt an OF style match first */ + if (of_driver_match_device(dev, drv)) + return 1; - return NULL; + /* Then try ACPI style match */ + if (acpi_driver_match_device(dev, drv)) + return 1; + + /* Then try to match against the id table */ + if (pdrv->id_table) + return platform_match_id(pdrv->id_table, pdev) != NULL; + + /* fall-back to driver name match */ + return (strcmp(pdev->name, drv->name) == 0); } -/** - * early_platform_left - check if early platform driver has matching devices - * @epdrv: early platform driver structure - * @id: return true if id or above exists - */ -static __init int early_platform_left(struct early_platform_driver *epdrv, - int id) +static int platform_uevent(const struct device *dev, struct kobj_uevent_env *env) { - struct platform_device *pd; + const struct platform_device *pdev = to_platform_device(dev); + int rc; + + /* Some devices have extra OF data and an OF-style MODALIAS */ + rc = of_device_uevent_modalias(dev, env); + if (rc != -ENODEV) + return rc; - list_for_each_entry(pd, &early_platform_device_list, dev.devres_head) - if (platform_match(&pd->dev, &epdrv->pdrv->driver)) - if (pd->id >= id) - return 1; + rc = acpi_device_uevent_modalias(dev, env); + if (rc != -ENODEV) + return rc; + add_uevent_var(env, "MODALIAS=%s%s", PLATFORM_MODULE_PREFIX, + pdev->name); return 0; } -/** - * early_platform_driver_probe_id - probe drivers matching class_str and id - * @class_str: string to identify early platform driver class - * @id: id to match against - * @nr_probe: number of platform devices to successfully probe before exiting - */ -static int __init early_platform_driver_probe_id(char *class_str, - int id, - int nr_probe) +static int platform_probe(struct device *_dev) { - struct early_platform_driver *epdrv; - struct platform_device *match; - int match_id; - int n = 0; - int left = 0; - - list_for_each_entry(epdrv, &early_platform_driver_list, list) { - /* only use drivers matching our class_str */ - if (strcmp(class_str, epdrv->class_str)) - continue; + struct platform_driver *drv = to_platform_driver(_dev->driver); + struct platform_device *dev = to_platform_device(_dev); + int ret; - if (id == -2) { - match_id = epdrv->requested_id; - left = 1; - - } else { - match_id = id; - left += early_platform_left(epdrv, id); - - /* skip requested id */ - switch (epdrv->requested_id) { - case EARLY_PLATFORM_ID_ERROR: - case EARLY_PLATFORM_ID_UNSET: - break; - default: - if (epdrv->requested_id == id) - match_id = EARLY_PLATFORM_ID_UNSET; - } - } + /* + * A driver registered using platform_driver_probe() cannot be bound + * again later because the probe function usually lives in __init code + * and so is gone. For these drivers .probe is set to + * platform_probe_fail in __platform_driver_probe(). Don't even prepare + * clocks and PM domains for these to match the traditional behaviour. + */ + if (unlikely(drv->probe == platform_probe_fail)) + return -ENXIO; - switch (match_id) { - case EARLY_PLATFORM_ID_ERROR: - pr_warn("%s: unable to parse %s parameter\n", - class_str, epdrv->pdrv->driver.name); - /* fall-through */ - case EARLY_PLATFORM_ID_UNSET: - match = NULL; - break; - default: - match = early_platform_match(epdrv, match_id); - } + ret = of_clk_set_defaults(_dev->of_node, false); + if (ret < 0) + return ret; - if (match) { - /* - * Set up a sensible init_name to enable - * dev_name() and others to be used before the - * rest of the driver core is initialized. - */ - if (!match->dev.init_name && slab_is_available()) { - if (match->id != -1) - match->dev.init_name = - kasprintf(GFP_KERNEL, "%s.%d", - match->name, - match->id); - else - match->dev.init_name = - kasprintf(GFP_KERNEL, "%s", - match->name); - - if (!match->dev.init_name) - return -ENOMEM; - } + ret = dev_pm_domain_attach(_dev, PD_FLAG_ATTACH_POWER_ON | + PD_FLAG_DETACH_POWER_OFF); + if (ret) + goto out; - if (epdrv->pdrv->probe(match)) - pr_warn("%s: unable to probe %s early.\n", - class_str, match->name); - else - n++; - } + if (drv->probe) + ret = drv->probe(dev); - if (n >= nr_probe) - break; +out: + if (drv->prevent_deferred_probe && ret == -EPROBE_DEFER) { + dev_warn(_dev, "probe deferral not supported\n"); + ret = -ENXIO; } - if (left) - return n; - else - return -ENODEV; + return ret; } -/** - * early_platform_driver_probe - probe a class of registered drivers - * @class_str: string to identify early platform driver class - * @nr_probe: number of platform devices to successfully probe before exiting - * @user_only: only probe user specified early platform devices - * - * Used by architecture code to probe registered early platform drivers - * within a certain class. For probe to happen a registered early platform - * device matching a registered early platform driver is needed. - */ -int __init early_platform_driver_probe(char *class_str, - int nr_probe, - int user_only) +static void platform_remove(struct device *_dev) { - int k, n, i; + struct platform_driver *drv = to_platform_driver(_dev->driver); + struct platform_device *dev = to_platform_device(_dev); - n = 0; - for (i = -2; n < nr_probe; i++) { - k = early_platform_driver_probe_id(class_str, i, nr_probe - n); + if (drv->remove) + drv->remove(dev); +} - if (k < 0) - break; +static void platform_shutdown(struct device *_dev) +{ + struct platform_device *dev = to_platform_device(_dev); + struct platform_driver *drv; - n += k; + if (!_dev->driver) + return; - if (user_only) - break; + drv = to_platform_driver(_dev->driver); + if (drv->shutdown) + drv->shutdown(dev); +} + +static int platform_dma_configure(struct device *dev) +{ + struct device_driver *drv = READ_ONCE(dev->driver); + struct fwnode_handle *fwnode = dev_fwnode(dev); + enum dev_dma_attr attr; + int ret = 0; + + if (is_of_node(fwnode)) { + ret = of_dma_configure(dev, to_of_node(fwnode), true); + } else if (is_acpi_device_node(fwnode)) { + attr = acpi_get_dma_attr(to_acpi_device_node(fwnode)); + ret = acpi_dma_configure(dev, attr); } + /* @dev->driver may not be valid when we're called from the IOMMU layer */ + if (ret || !drv || to_platform_driver(drv)->driver_managed_dma) + return ret; + + ret = iommu_device_use_default_domain(dev); + if (ret) + arch_teardown_dma_ops(dev); - return n; + return ret; +} + +static void platform_dma_cleanup(struct device *dev) +{ + struct platform_driver *drv = to_platform_driver(dev->driver); + + if (!drv->driver_managed_dma) + iommu_device_unuse_default_domain(dev); +} + +static const struct dev_pm_ops platform_dev_pm_ops = { + SET_RUNTIME_PM_OPS(pm_generic_runtime_suspend, pm_generic_runtime_resume, NULL) + USE_PLATFORM_PM_SLEEP_OPS +}; + +const struct bus_type platform_bus_type = { + .name = "platform", + .dev_groups = platform_dev_groups, + .match = platform_match, + .uevent = platform_uevent, + .probe = platform_probe, + .remove = platform_remove, + .shutdown = platform_shutdown, + .dma_configure = platform_dma_configure, + .dma_cleanup = platform_dma_cleanup, + .pm = &platform_dev_pm_ops, +}; +EXPORT_SYMBOL_GPL(platform_bus_type); + +static inline int __platform_match(struct device *dev, const void *drv) +{ + return platform_match(dev, (struct device_driver *)drv); } /** - * early_platform_cleanup - clean up early platform code + * platform_find_device_by_driver - Find a platform device with a given + * driver. + * @start: The device to start the search from. + * @drv: The device driver to look for. */ -void __init early_platform_cleanup(void) +struct device *platform_find_device_by_driver(struct device *start, + const struct device_driver *drv) { - struct platform_device *pd, *pd2; + return bus_find_device(&platform_bus_type, start, drv, + __platform_match); +} +EXPORT_SYMBOL_GPL(platform_find_device_by_driver); - /* clean up the devres list used to chain devices */ - list_for_each_entry_safe(pd, pd2, &early_platform_device_list, - dev.devres_head) { - list_del(&pd->dev.devres_head); - memset(&pd->dev.devres_head, 0, sizeof(pd->dev.devres_head)); +void __weak __init early_platform_cleanup(void) { } + +int __init platform_bus_init(void) +{ + int error; + + early_platform_cleanup(); + + error = device_register(&platform_bus); + if (error) { + put_device(&platform_bus); + return error; } -} + error = bus_register(&platform_bus_type); + if (error) + device_unregister(&platform_bus); + return error; +} diff --git a/drivers/base/power/Makefile b/drivers/base/power/Makefile index 2e58ebb1f6c0..2989e42d0161 100644 --- a/drivers/base/power/Makefile +++ b/drivers/base/power/Makefile @@ -1,9 +1,9 @@ -obj-$(CONFIG_PM) += sysfs.o generic_ops.o common.o qos.o -obj-$(CONFIG_PM_SLEEP) += main.o wakeup.o -obj-$(CONFIG_PM_RUNTIME) += runtime.o +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_PM) += sysfs.o generic_ops.o common.o qos.o runtime.o wakeirq.o +obj-$(CONFIG_PM_SLEEP) += main.o wakeup.o wakeup_stats.o obj-$(CONFIG_PM_TRACE_RTC) += trace.o -obj-$(CONFIG_PM_OPP) += opp.o -obj-$(CONFIG_PM_GENERIC_DOMAINS) += domain.o domain_governor.o obj-$(CONFIG_HAVE_CLK) += clock_ops.o +obj-$(CONFIG_PM_QOS_KUNIT_TEST) += qos-test.o +obj-$(CONFIG_PM_RUNTIME_KUNIT_TEST) += runtime-test.o ccflags-$(CONFIG_DEBUG_DRIVER) := -DDEBUG diff --git a/drivers/base/power/clock_ops.c b/drivers/base/power/clock_ops.c index 9d8fde709390..b69bcb37c830 100644 --- a/drivers/base/power/clock_ops.c +++ b/drivers/base/power/clock_ops.c @@ -1,26 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0 /* * drivers/base/power/clock_ops.c - Generic clock manipulation PM callbacks * * Copyright (c) 2011 Rafael J. Wysocki <rjw@sisk.pl>, Renesas Electronics Corp. - * - * This file is released under the GPLv2. */ -#include <linux/init.h> #include <linux/kernel.h> #include <linux/device.h> #include <linux/io.h> #include <linux/pm.h> #include <linux/pm_clock.h> #include <linux/clk.h> +#include <linux/clkdev.h> +#include <linux/of_clk.h> #include <linux/slab.h> #include <linux/err.h> +#include <linux/pm_domain.h> +#include <linux/pm_runtime.h> -#ifdef CONFIG_PM +#ifdef CONFIG_PM_CLK enum pce_status { PCE_STATUS_NONE = 0, PCE_STATUS_ACQUIRED, + PCE_STATUS_PREPARED, PCE_STATUS_ENABLED, PCE_STATUS_ERROR, }; @@ -30,33 +33,167 @@ struct pm_clock_entry { char *con_id; struct clk *clk; enum pce_status status; + bool enabled_when_prepared; }; /** + * pm_clk_list_lock - ensure exclusive access for modifying the PM clock + * entry list. + * @psd: pm_subsys_data instance corresponding to the PM clock entry list + * and clk_op_might_sleep count to be modified. + * + * Get exclusive access before modifying the PM clock entry list and the + * clock_op_might_sleep count to guard against concurrent modifications. + * This also protects against a concurrent clock_op_might_sleep and PM clock + * entry list usage in pm_clk_suspend()/pm_clk_resume() that may or may not + * happen in atomic context, hence both the mutex and the spinlock must be + * taken here. + */ +static void pm_clk_list_lock(struct pm_subsys_data *psd) + __acquires(&psd->lock) +{ + mutex_lock(&psd->clock_mutex); + spin_lock_irq(&psd->lock); +} + +/** + * pm_clk_list_unlock - counterpart to pm_clk_list_lock(). + * @psd: the same pm_subsys_data instance previously passed to + * pm_clk_list_lock(). + */ +static void pm_clk_list_unlock(struct pm_subsys_data *psd) + __releases(&psd->lock) +{ + spin_unlock_irq(&psd->lock); + mutex_unlock(&psd->clock_mutex); +} + +/** + * pm_clk_op_lock - ensure exclusive access for performing clock operations. + * @psd: pm_subsys_data instance corresponding to the PM clock entry list + * and clk_op_might_sleep count being used. + * @flags: stored irq flags. + * @fn: string for the caller function's name. + * + * This is used by pm_clk_suspend() and pm_clk_resume() to guard + * against concurrent modifications to the clock entry list and the + * clock_op_might_sleep count. If clock_op_might_sleep is != 0 then + * only the mutex can be locked and those functions can only be used in + * non atomic context. If clock_op_might_sleep == 0 then these functions + * may be used in any context and only the spinlock can be locked. + * Returns -EINVAL if called in atomic context when clock ops might sleep. + */ +static int pm_clk_op_lock(struct pm_subsys_data *psd, unsigned long *flags, + const char *fn) + /* sparse annotations don't work here as exit state isn't static */ +{ + bool atomic_context = in_atomic() || irqs_disabled(); + +try_again: + spin_lock_irqsave(&psd->lock, *flags); + if (!psd->clock_op_might_sleep) { + /* the __release is there to work around sparse limitations */ + __release(&psd->lock); + return 0; + } + + /* bail out if in atomic context */ + if (atomic_context) { + pr_err("%s: atomic context with clock_ops_might_sleep = %d", + fn, psd->clock_op_might_sleep); + spin_unlock_irqrestore(&psd->lock, *flags); + might_sleep(); + return -EPERM; + } + + /* we must switch to the mutex */ + spin_unlock_irqrestore(&psd->lock, *flags); + mutex_lock(&psd->clock_mutex); + + /* + * There was a possibility for psd->clock_op_might_sleep + * to become 0 above. Keep the mutex only if not the case. + */ + if (likely(psd->clock_op_might_sleep)) + return 0; + + mutex_unlock(&psd->clock_mutex); + goto try_again; +} + +/** + * pm_clk_op_unlock - counterpart to pm_clk_op_lock(). + * @psd: the same pm_subsys_data instance previously passed to + * pm_clk_op_lock(). + * @flags: irq flags provided by pm_clk_op_lock(). + */ +static void pm_clk_op_unlock(struct pm_subsys_data *psd, unsigned long *flags) + /* sparse annotations don't work here as entry state isn't static */ +{ + if (psd->clock_op_might_sleep) { + mutex_unlock(&psd->clock_mutex); + } else { + /* the __acquire is there to work around sparse limitations */ + __acquire(&psd->lock); + spin_unlock_irqrestore(&psd->lock, *flags); + } +} + +/** + * __pm_clk_enable - Enable a clock, reporting any errors + * @dev: The device for the given clock + * @ce: PM clock entry corresponding to the clock. + */ +static inline void __pm_clk_enable(struct device *dev, struct pm_clock_entry *ce) +{ + int ret; + + switch (ce->status) { + case PCE_STATUS_ACQUIRED: + ret = clk_prepare_enable(ce->clk); + break; + case PCE_STATUS_PREPARED: + ret = clk_enable(ce->clk); + break; + default: + return; + } + if (!ret) + ce->status = PCE_STATUS_ENABLED; + else + dev_err(dev, "%s: failed to enable clk %p, error %d\n", + __func__, ce->clk, ret); +} + +/** * pm_clk_acquire - Acquire a device clock. * @dev: Device whose clock is to be acquired. * @ce: PM clock entry corresponding to the clock. */ static void pm_clk_acquire(struct device *dev, struct pm_clock_entry *ce) { - ce->clk = clk_get(dev, ce->con_id); + if (!ce->clk) + ce->clk = clk_get(dev, ce->con_id); if (IS_ERR(ce->clk)) { ce->status = PCE_STATUS_ERROR; - } else { + return; + } else if (clk_is_enabled_when_prepared(ce->clk)) { + /* we defer preparing the clock in that case */ ce->status = PCE_STATUS_ACQUIRED; - dev_dbg(dev, "Clock %s managed by runtime PM.\n", ce->con_id); + ce->enabled_when_prepared = true; + } else if (clk_prepare(ce->clk)) { + ce->status = PCE_STATUS_ERROR; + dev_err(dev, "clk_prepare() failed\n"); + return; + } else { + ce->status = PCE_STATUS_PREPARED; } + dev_dbg(dev, "Clock %pC con_id %s managed by runtime PM.\n", + ce->clk, ce->con_id); } -/** - * pm_clk_add - Start using a device clock for power management. - * @dev: Device whose clock is going to be used for power management. - * @con_id: Connection ID of the clock. - * - * Add the clock represented by @con_id to the list of clocks used for - * the power management of @dev. - */ -int pm_clk_add(struct device *dev, const char *con_id) +static int __pm_clk_add(struct device *dev, const char *con_id, + struct clk *clk) { struct pm_subsys_data *psd = dev_to_psd(dev); struct pm_clock_entry *ce; @@ -65,30 +202,119 @@ int pm_clk_add(struct device *dev, const char *con_id) return -EINVAL; ce = kzalloc(sizeof(*ce), GFP_KERNEL); - if (!ce) { - dev_err(dev, "Not enough memory for clock entry.\n"); + if (!ce) return -ENOMEM; - } if (con_id) { ce->con_id = kstrdup(con_id, GFP_KERNEL); if (!ce->con_id) { - dev_err(dev, - "Not enough memory for clock connection ID.\n"); kfree(ce); return -ENOMEM; } + } else { + if (IS_ERR(clk)) { + kfree(ce); + return -ENOENT; + } + ce->clk = clk; } pm_clk_acquire(dev, ce); - spin_lock_irq(&psd->lock); + pm_clk_list_lock(psd); list_add_tail(&ce->node, &psd->clock_list); - spin_unlock_irq(&psd->lock); + if (ce->enabled_when_prepared) + psd->clock_op_might_sleep++; + pm_clk_list_unlock(psd); return 0; } /** + * pm_clk_add - Start using a device clock for power management. + * @dev: Device whose clock is going to be used for power management. + * @con_id: Connection ID of the clock. + * + * Add the clock represented by @con_id to the list of clocks used for + * the power management of @dev. + */ +int pm_clk_add(struct device *dev, const char *con_id) +{ + return __pm_clk_add(dev, con_id, NULL); +} +EXPORT_SYMBOL_GPL(pm_clk_add); + +/** + * pm_clk_add_clk - Start using a device clock for power management. + * @dev: Device whose clock is going to be used for power management. + * @clk: Clock pointer + * + * Add the clock to the list of clocks used for the power management of @dev. + * The power-management code will take control of the clock reference, so + * callers should not call clk_put() on @clk after this function sucessfully + * returned. + */ +int pm_clk_add_clk(struct device *dev, struct clk *clk) +{ + return __pm_clk_add(dev, NULL, clk); +} +EXPORT_SYMBOL_GPL(pm_clk_add_clk); + +/** + * of_pm_clk_add_clks - Start using device clock(s) for power management. + * @dev: Device whose clock(s) is going to be used for power management. + * + * Add a series of clocks described in the 'clocks' device-tree node for + * a device to the list of clocks used for the power management of @dev. + * On success, returns the number of clocks added. Returns a negative + * error code if there are no clocks in the device node for the device + * or if adding a clock fails. + */ +int of_pm_clk_add_clks(struct device *dev) +{ + struct clk **clks; + int i, count; + int ret; + + if (!dev || !dev->of_node) + return -EINVAL; + + count = of_clk_get_parent_count(dev->of_node); + if (count <= 0) + return -ENODEV; + + clks = kcalloc(count, sizeof(*clks), GFP_KERNEL); + if (!clks) + return -ENOMEM; + + for (i = 0; i < count; i++) { + clks[i] = of_clk_get(dev->of_node, i); + if (IS_ERR(clks[i])) { + ret = PTR_ERR(clks[i]); + goto error; + } + + ret = pm_clk_add_clk(dev, clks[i]); + if (ret) { + clk_put(clks[i]); + goto error; + } + } + + kfree(clks); + + return i; + +error: + while (i--) + pm_clk_remove_clk(dev, clks[i]); + + kfree(clks); + + return ret; +} +EXPORT_SYMBOL_GPL(of_pm_clk_add_clks); + +/** * __pm_clk_remove - Destroy PM clock entry. * @ce: PM clock entry to destroy. */ @@ -97,12 +323,20 @@ static void __pm_clk_remove(struct pm_clock_entry *ce) if (!ce) return; - if (ce->status < PCE_STATUS_ERROR) { - if (ce->status == PCE_STATUS_ENABLED) - clk_disable_unprepare(ce->clk); - - if (ce->status >= PCE_STATUS_ACQUIRED) + switch (ce->status) { + case PCE_STATUS_ENABLED: + clk_disable(ce->clk); + fallthrough; + case PCE_STATUS_PREPARED: + clk_unprepare(ce->clk); + fallthrough; + case PCE_STATUS_ACQUIRED: + case PCE_STATUS_ERROR: + if (!IS_ERR(ce->clk)) clk_put(ce->clk); + break; + default: + break; } kfree(ce->con_id); @@ -110,55 +344,58 @@ static void __pm_clk_remove(struct pm_clock_entry *ce) } /** - * pm_clk_remove - Stop using a device clock for power management. + * pm_clk_remove_clk - Stop using a device clock for power management. * @dev: Device whose clock should not be used for PM any more. - * @con_id: Connection ID of the clock. + * @clk: Clock pointer * - * Remove the clock represented by @con_id from the list of clocks used for + * Remove the clock pointed to by @clk from the list of clocks used for * the power management of @dev. */ -void pm_clk_remove(struct device *dev, const char *con_id) +void pm_clk_remove_clk(struct device *dev, struct clk *clk) { struct pm_subsys_data *psd = dev_to_psd(dev); struct pm_clock_entry *ce; - if (!psd) + if (!psd || !clk) return; - spin_lock_irq(&psd->lock); + pm_clk_list_lock(psd); list_for_each_entry(ce, &psd->clock_list, node) { - if (!con_id && !ce->con_id) - goto remove; - else if (!con_id || !ce->con_id) - continue; - else if (!strcmp(con_id, ce->con_id)) + if (clk == ce->clk) goto remove; } - spin_unlock_irq(&psd->lock); + pm_clk_list_unlock(psd); return; remove: list_del(&ce->node); - spin_unlock_irq(&psd->lock); + if (ce->enabled_when_prepared) + psd->clock_op_might_sleep--; + pm_clk_list_unlock(psd); __pm_clk_remove(ce); } +EXPORT_SYMBOL_GPL(pm_clk_remove_clk); /** * pm_clk_init - Initialize a device's list of power management clocks. * @dev: Device to initialize the list of PM clocks for. * * Initialize the lock and clock_list members of the device's pm_subsys_data - * object. + * object, set the count of clocks that might sleep to 0. */ void pm_clk_init(struct device *dev) { struct pm_subsys_data *psd = dev_to_psd(dev); - if (psd) + if (psd) { INIT_LIST_HEAD(&psd->clock_list); + mutex_init(&psd->clock_mutex); + psd->clock_op_might_sleep = 0; + } } +EXPORT_SYMBOL_GPL(pm_clk_init); /** * pm_clk_create - Create and initialize a device's list of PM clocks. @@ -171,6 +408,7 @@ int pm_clk_create(struct device *dev) { return dev_pm_get_subsys_data(dev); } +EXPORT_SYMBOL_GPL(pm_clk_create); /** * pm_clk_destroy - Destroy a device's list of power management clocks. @@ -191,12 +429,13 @@ void pm_clk_destroy(struct device *dev) INIT_LIST_HEAD(&list); - spin_lock_irq(&psd->lock); + pm_clk_list_lock(psd); list_for_each_entry_safe_reverse(ce, c, &psd->clock_list, node) list_move(&ce->node, &list); + psd->clock_op_might_sleep = 0; - spin_unlock_irq(&psd->lock); + pm_clk_list_unlock(psd); dev_pm_put_subsys_data(dev); @@ -205,10 +444,24 @@ void pm_clk_destroy(struct device *dev) __pm_clk_remove(ce); } } +EXPORT_SYMBOL_GPL(pm_clk_destroy); + +static void pm_clk_destroy_action(void *data) +{ + pm_clk_destroy(data); +} + +int devm_pm_clk_create(struct device *dev) +{ + int ret; -#endif /* CONFIG_PM */ + ret = pm_clk_create(dev); + if (ret) + return ret; -#ifdef CONFIG_PM_RUNTIME + return devm_add_action_or_reset(dev, pm_clk_destroy_action, dev); +} +EXPORT_SYMBOL_GPL(devm_pm_clk_create); /** * pm_clk_suspend - Disable clocks in a device's PM clock list. @@ -219,26 +472,34 @@ int pm_clk_suspend(struct device *dev) struct pm_subsys_data *psd = dev_to_psd(dev); struct pm_clock_entry *ce; unsigned long flags; + int ret; dev_dbg(dev, "%s()\n", __func__); if (!psd) return 0; - spin_lock_irqsave(&psd->lock, flags); + ret = pm_clk_op_lock(psd, &flags, __func__); + if (ret) + return ret; list_for_each_entry_reverse(ce, &psd->clock_list, node) { - if (ce->status < PCE_STATUS_ERROR) { - if (ce->status == PCE_STATUS_ENABLED) + if (ce->status == PCE_STATUS_ENABLED) { + if (ce->enabled_when_prepared) { + clk_disable_unprepare(ce->clk); + ce->status = PCE_STATUS_ACQUIRED; + } else { clk_disable(ce->clk); - ce->status = PCE_STATUS_ACQUIRED; + ce->status = PCE_STATUS_PREPARED; + } } } - spin_unlock_irqrestore(&psd->lock, flags); + pm_clk_op_unlock(psd, &flags); return 0; } +EXPORT_SYMBOL_GPL(pm_clk_suspend); /** * pm_clk_resume - Enable clocks in a device's PM clock list. @@ -249,25 +510,25 @@ int pm_clk_resume(struct device *dev) struct pm_subsys_data *psd = dev_to_psd(dev); struct pm_clock_entry *ce; unsigned long flags; + int ret; dev_dbg(dev, "%s()\n", __func__); if (!psd) return 0; - spin_lock_irqsave(&psd->lock, flags); + ret = pm_clk_op_lock(psd, &flags, __func__); + if (ret) + return ret; - list_for_each_entry(ce, &psd->clock_list, node) { - if (ce->status < PCE_STATUS_ERROR) { - clk_enable(ce->clk); - ce->status = PCE_STATUS_ENABLED; - } - } + list_for_each_entry(ce, &psd->clock_list, node) + __pm_clk_enable(dev, ce); - spin_unlock_irqrestore(&psd->lock, flags); + pm_clk_op_unlock(psd, &flags); return 0; } +EXPORT_SYMBOL_GPL(pm_clk_resume); /** * pm_clk_notify - Notify routine for device addition and removal. @@ -306,7 +567,7 @@ static int pm_clk_notify(struct notifier_block *nb, if (error) break; - dev->pm_domain = clknb->pm_domain; + dev_pm_domain_set(dev, clknb->pm_domain); if (clknb->con_ids[0]) { for (con_id = clknb->con_ids; *con_id; con_id++) pm_clk_add(dev, *con_id); @@ -319,7 +580,7 @@ static int pm_clk_notify(struct notifier_block *nb, if (dev->pm_domain != clknb->pm_domain) break; - dev->pm_domain = NULL; + dev_pm_domain_set(dev, NULL); pm_clk_destroy(dev); break; } @@ -327,63 +588,46 @@ static int pm_clk_notify(struct notifier_block *nb, return 0; } -#else /* !CONFIG_PM_RUNTIME */ - -#ifdef CONFIG_PM - -/** - * pm_clk_suspend - Disable clocks in a device's PM clock list. - * @dev: Device to disable the clocks for. - */ -int pm_clk_suspend(struct device *dev) +int pm_clk_runtime_suspend(struct device *dev) { - struct pm_subsys_data *psd = dev_to_psd(dev); - struct pm_clock_entry *ce; - unsigned long flags; + int ret; - dev_dbg(dev, "%s()\n", __func__); - - /* If there is no driver, the clocks are already disabled. */ - if (!psd || !dev->driver) - return 0; + dev_dbg(dev, "%s\n", __func__); - spin_lock_irqsave(&psd->lock, flags); - - list_for_each_entry_reverse(ce, &psd->clock_list, node) - clk_disable(ce->clk); + ret = pm_generic_runtime_suspend(dev); + if (ret) { + dev_err(dev, "failed to suspend device\n"); + return ret; + } - spin_unlock_irqrestore(&psd->lock, flags); + ret = pm_clk_suspend(dev); + if (ret) { + dev_err(dev, "failed to suspend clock\n"); + pm_generic_runtime_resume(dev); + return ret; + } return 0; } +EXPORT_SYMBOL_GPL(pm_clk_runtime_suspend); -/** - * pm_clk_resume - Enable clocks in a device's PM clock list. - * @dev: Device to enable the clocks for. - */ -int pm_clk_resume(struct device *dev) +int pm_clk_runtime_resume(struct device *dev) { - struct pm_subsys_data *psd = dev_to_psd(dev); - struct pm_clock_entry *ce; - unsigned long flags; - - dev_dbg(dev, "%s()\n", __func__); - - /* If there is no driver, the clocks should remain disabled. */ - if (!psd || !dev->driver) - return 0; - - spin_lock_irqsave(&psd->lock, flags); + int ret; - list_for_each_entry(ce, &psd->clock_list, node) - clk_enable(ce->clk); + dev_dbg(dev, "%s\n", __func__); - spin_unlock_irqrestore(&psd->lock, flags); + ret = pm_clk_resume(dev); + if (ret) { + dev_err(dev, "failed to resume clock\n"); + return ret; + } - return 0; + return pm_generic_runtime_resume(dev); } +EXPORT_SYMBOL_GPL(pm_clk_runtime_resume); -#endif /* CONFIG_PM */ +#else /* !CONFIG_PM_CLK */ /** * enable_clock - Enable a device clock. @@ -450,6 +694,7 @@ static int pm_clk_notify(struct notifier_block *nb, enable_clock(dev, NULL); } break; + case BUS_NOTIFY_DRIVER_NOT_BOUND: case BUS_NOTIFY_UNBOUND_DRIVER: if (clknb->con_ids[0]) { for (con_id = clknb->con_ids; *con_id; con_id++) @@ -463,7 +708,7 @@ static int pm_clk_notify(struct notifier_block *nb, return 0; } -#endif /* !CONFIG_PM_RUNTIME */ +#endif /* !CONFIG_PM_CLK */ /** * pm_clk_add_notifier - Add bus type notifier for power management clocks. @@ -475,7 +720,7 @@ static int pm_clk_notify(struct notifier_block *nb, * the remaining members of @clknb should be populated prior to calling this * routine. */ -void pm_clk_add_notifier(struct bus_type *bus, +void pm_clk_add_notifier(const struct bus_type *bus, struct pm_clk_notifier_block *clknb) { if (!bus || !clknb) @@ -484,3 +729,4 @@ void pm_clk_add_notifier(struct bus_type *bus, clknb->nb.notifier_call = pm_clk_notify; bus_register_notifier(bus, &clknb->nb); } +EXPORT_SYMBOL_GPL(pm_clk_add_notifier); diff --git a/drivers/base/power/common.c b/drivers/base/power/common.c index 5da914041305..6ecf9ce4a4e6 100644 --- a/drivers/base/power/common.c +++ b/drivers/base/power/common.c @@ -1,25 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0 /* * drivers/base/power/common.c - Common device power management code. * * Copyright (C) 2011 Rafael J. Wysocki <rjw@sisk.pl>, Renesas Electronics Corp. - * - * This file is released under the GPLv2. */ - -#include <linux/init.h> #include <linux/kernel.h> #include <linux/device.h> #include <linux/export.h> #include <linux/slab.h> #include <linux/pm_clock.h> +#include <linux/acpi.h> +#include <linux/pm_domain.h> +#include <linux/pm_opp.h> + +#include "power.h" /** * dev_pm_get_subsys_data - Create or refcount power.subsys_data for device. * @dev: Device to handle. * * If power.subsys_data is NULL, point it to a new object, otherwise increment - * its reference counter. Return 1 if a new object has been created, otherwise - * return 0 or error code. + * its reference counter. Return 0 if new object has been created or refcount + * increased, otherwise negative error code. */ int dev_pm_get_subsys_data(struct device *dev) { @@ -55,13 +57,11 @@ EXPORT_SYMBOL_GPL(dev_pm_get_subsys_data); * @dev: Device to handle. * * If the reference counter of power.subsys_data is zero after dropping the - * reference, power.subsys_data is removed. Return 1 if that happens or 0 - * otherwise. + * reference, power.subsys_data is removed. */ -int dev_pm_put_subsys_data(struct device *dev) +void dev_pm_put_subsys_data(struct device *dev) { struct pm_subsys_data *psd; - int ret = 1; spin_lock_irq(&dev->power.lock); @@ -69,17 +69,389 @@ int dev_pm_put_subsys_data(struct device *dev) if (!psd) goto out; - if (--psd->refcount == 0) { + if (--psd->refcount == 0) dev->power.subsys_data = NULL; - } else { + else psd = NULL; - ret = 0; - } out: spin_unlock_irq(&dev->power.lock); kfree(psd); +} +EXPORT_SYMBOL_GPL(dev_pm_put_subsys_data); + +/** + * dev_pm_domain_attach - Attach a device to its PM domain. + * @dev: Device to attach. + * @flags: indicate whether we should power on/off the device on attach/detach + * + * The @dev may only be attached to a single PM domain. By iterating through + * the available alternatives we try to find a valid PM domain for the device. + * As attachment succeeds, the ->detach() callback in the struct dev_pm_domain + * should be assigned by the corresponding attach function. + * + * This function should typically be invoked from subsystem level code during + * the probe phase. Especially for those that holds devices which requires + * power management through PM domains. + * + * Callers must ensure proper synchronization of this function with power + * management callbacks. + * + * Returns 0 on successfully attached PM domain, or when it is found that the + * device doesn't need a PM domain, else a negative error code. + */ +int dev_pm_domain_attach(struct device *dev, u32 flags) +{ + int ret; + + if (dev->pm_domain) + return 0; + + ret = acpi_dev_pm_attach(dev, !!(flags & PD_FLAG_ATTACH_POWER_ON)); + if (!ret) + ret = genpd_dev_pm_attach(dev); + + if (dev->pm_domain) + dev->power.detach_power_off = !!(flags & PD_FLAG_DETACH_POWER_OFF); + + return ret < 0 ? ret : 0; +} +EXPORT_SYMBOL_GPL(dev_pm_domain_attach); + +/** + * dev_pm_domain_attach_by_id - Associate a device with one of its PM domains. + * @dev: The device used to lookup the PM domain. + * @index: The index of the PM domain. + * + * As @dev may only be attached to a single PM domain, the backend PM domain + * provider creates a virtual device to attach instead. If attachment succeeds, + * the ->detach() callback in the struct dev_pm_domain are assigned by the + * corresponding backend attach function, as to deal with detaching of the + * created virtual device. + * + * This function should typically be invoked by a driver during the probe phase, + * in case its device requires power management through multiple PM domains. The + * driver may benefit from using the received device, to configure device-links + * towards its original device. Depending on the use-case and if needed, the + * links may be dynamically changed by the driver, which allows it to control + * the power to the PM domains independently from each other. + * + * Callers must ensure proper synchronization of this function with power + * management callbacks. + * + * Returns the virtual created device when successfully attached to its PM + * domain, NULL in case @dev don't need a PM domain, else an ERR_PTR(). + * Note that, to detach the returned virtual device, the driver shall call + * dev_pm_domain_detach() on it, typically during the remove phase. + */ +struct device *dev_pm_domain_attach_by_id(struct device *dev, + unsigned int index) +{ + if (dev->pm_domain) + return ERR_PTR(-EEXIST); + + return genpd_dev_pm_attach_by_id(dev, index); +} +EXPORT_SYMBOL_GPL(dev_pm_domain_attach_by_id); + +/** + * dev_pm_domain_attach_by_name - Associate a device with one of its PM domains. + * @dev: The device used to lookup the PM domain. + * @name: The name of the PM domain. + * + * For a detailed function description, see dev_pm_domain_attach_by_id(). + */ +struct device *dev_pm_domain_attach_by_name(struct device *dev, + const char *name) +{ + if (dev->pm_domain) + return ERR_PTR(-EEXIST); + + return genpd_dev_pm_attach_by_name(dev, name); +} +EXPORT_SYMBOL_GPL(dev_pm_domain_attach_by_name); + +/** + * dev_pm_domain_attach_list - Associate a device with its PM domains. + * @dev: The device used to lookup the PM domains for. + * @data: The data used for attaching to the PM domains. + * @list: An out-parameter with an allocated list of attached PM domains. + * + * This function helps to attach a device to its multiple PM domains. The + * caller, which is typically a driver's probe function, may provide a list of + * names for the PM domains that we should try to attach the device to, but it + * may also provide an empty list, in case the attach should be done for all of + * the available PM domains. + * + * Callers must ensure proper synchronization of this function with power + * management callbacks. + * + * Returns the number of attached PM domains or a negative error code in case of + * a failure. Note that, to detach the list of PM domains, the driver shall call + * dev_pm_domain_detach_list(), typically during the remove phase. + */ +int dev_pm_domain_attach_list(struct device *dev, + const struct dev_pm_domain_attach_data *data, + struct dev_pm_domain_list **list) +{ + struct device_node *np = dev->of_node; + struct dev_pm_domain_list *pds; + struct device *pd_dev = NULL; + int ret, i, num_pds = 0; + bool by_id = true; + size_t size; + u32 pd_flags = data ? data->pd_flags : 0; + u32 link_flags = pd_flags & PD_FLAG_NO_DEV_LINK ? 0 : + DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME; + + if (dev->pm_domain) + return -EEXIST; + + /* For now this is limited to OF based platforms. */ + if (!np) + return 0; + + if (data && data->pd_names) { + num_pds = data->num_pd_names; + by_id = false; + } else { + num_pds = of_count_phandle_with_args(np, "power-domains", + "#power-domain-cells"); + } + + if (num_pds <= 0) + return 0; + + pds = kzalloc(sizeof(*pds), GFP_KERNEL); + if (!pds) + return -ENOMEM; + + size = sizeof(*pds->pd_devs) + sizeof(*pds->pd_links) + + sizeof(*pds->opp_tokens); + pds->pd_devs = kcalloc(num_pds, size, GFP_KERNEL); + if (!pds->pd_devs) { + ret = -ENOMEM; + goto free_pds; + } + pds->pd_links = (void *)(pds->pd_devs + num_pds); + pds->opp_tokens = (void *)(pds->pd_links + num_pds); + + if (link_flags && pd_flags & PD_FLAG_DEV_LINK_ON) + link_flags |= DL_FLAG_RPM_ACTIVE; + + for (i = 0; i < num_pds; i++) { + if (by_id) + pd_dev = dev_pm_domain_attach_by_id(dev, i); + else + pd_dev = dev_pm_domain_attach_by_name(dev, + data->pd_names[i]); + if (IS_ERR_OR_NULL(pd_dev)) { + ret = pd_dev ? PTR_ERR(pd_dev) : -ENODEV; + goto err_attach; + } + + if (pd_flags & PD_FLAG_REQUIRED_OPP) { + struct dev_pm_opp_config config = { + .required_dev = pd_dev, + .required_dev_index = i, + }; + + ret = dev_pm_opp_set_config(dev, &config); + if (ret < 0) + goto err_link; + + pds->opp_tokens[i] = ret; + } + + if (link_flags) { + struct device_link *link; + + link = device_link_add(dev, pd_dev, link_flags); + if (!link) { + ret = -ENODEV; + goto err_link; + } + + pds->pd_links[i] = link; + } + + pds->pd_devs[i] = pd_dev; + } + pds->num_pds = num_pds; + *list = pds; + return num_pds; + +err_link: + dev_pm_opp_clear_config(pds->opp_tokens[i]); + dev_pm_domain_detach(pd_dev, true); +err_attach: + while (--i >= 0) { + dev_pm_opp_clear_config(pds->opp_tokens[i]); + if (pds->pd_links[i]) + device_link_del(pds->pd_links[i]); + dev_pm_domain_detach(pds->pd_devs[i], true); + } + kfree(pds->pd_devs); +free_pds: + kfree(pds); return ret; } -EXPORT_SYMBOL_GPL(dev_pm_put_subsys_data); +EXPORT_SYMBOL_GPL(dev_pm_domain_attach_list); + +/** + * devm_pm_domain_detach_list - devres-enabled version of dev_pm_domain_detach_list. + * @_list: The list of PM domains to detach. + * + * This function reverse the actions from devm_pm_domain_attach_list(). + * it will be invoked during the remove phase from drivers implicitly if driver + * uses devm_pm_domain_attach_list() to attach the PM domains. + */ +static void devm_pm_domain_detach_list(void *_list) +{ + struct dev_pm_domain_list *list = _list; + + dev_pm_domain_detach_list(list); +} + +/** + * devm_pm_domain_attach_list - devres-enabled version of dev_pm_domain_attach_list + * @dev: The device used to lookup the PM domains for. + * @data: The data used for attaching to the PM domains. + * @list: An out-parameter with an allocated list of attached PM domains. + * + * NOTE: this will also handle calling devm_pm_domain_detach_list() for + * you during remove phase. + * + * Returns the number of attached PM domains or a negative error code in case of + * a failure. + */ +int devm_pm_domain_attach_list(struct device *dev, + const struct dev_pm_domain_attach_data *data, + struct dev_pm_domain_list **list) +{ + int ret, num_pds; + + num_pds = dev_pm_domain_attach_list(dev, data, list); + if (num_pds <= 0) + return num_pds; + + ret = devm_add_action_or_reset(dev, devm_pm_domain_detach_list, *list); + if (ret) + return ret; + + return num_pds; +} +EXPORT_SYMBOL_GPL(devm_pm_domain_attach_list); + +/** + * dev_pm_domain_detach - Detach a device from its PM domain. + * @dev: Device to detach. + * @power_off: Used to indicate whether we should power off the device. + * + * This functions will reverse the actions from dev_pm_domain_attach(), + * dev_pm_domain_attach_by_id() and dev_pm_domain_attach_by_name(), thus it + * detaches @dev from its PM domain. Typically it should be invoked during the + * remove phase, either from subsystem level code or from drivers. + * + * Callers must ensure proper synchronization of this function with power + * management callbacks. + */ +void dev_pm_domain_detach(struct device *dev, bool power_off) +{ + if (dev->pm_domain && dev->pm_domain->detach) + dev->pm_domain->detach(dev, power_off); +} +EXPORT_SYMBOL_GPL(dev_pm_domain_detach); + +/** + * dev_pm_domain_detach_list - Detach a list of PM domains. + * @list: The list of PM domains to detach. + * + * This function reverse the actions from dev_pm_domain_attach_list(). + * Typically it should be invoked during the remove phase from drivers. + * + * Callers must ensure proper synchronization of this function with power + * management callbacks. + */ +void dev_pm_domain_detach_list(struct dev_pm_domain_list *list) +{ + int i; + + if (!list) + return; + + for (i = 0; i < list->num_pds; i++) { + dev_pm_opp_clear_config(list->opp_tokens[i]); + if (list->pd_links[i]) + device_link_del(list->pd_links[i]); + dev_pm_domain_detach(list->pd_devs[i], true); + } + + kfree(list->pd_devs); + kfree(list); +} +EXPORT_SYMBOL_GPL(dev_pm_domain_detach_list); + +/** + * dev_pm_domain_start - Start the device through its PM domain. + * @dev: Device to start. + * + * This function should typically be called during probe by a subsystem/driver, + * when it needs to start its device from the PM domain's perspective. Note + * that, it's assumed that the PM domain is already powered on when this + * function is called. + * + * Returns 0 on success and negative error values on failures. + */ +int dev_pm_domain_start(struct device *dev) +{ + if (dev->pm_domain && dev->pm_domain->start) + return dev->pm_domain->start(dev); + + return 0; +} +EXPORT_SYMBOL_GPL(dev_pm_domain_start); + +/** + * dev_pm_domain_set - Set PM domain of a device. + * @dev: Device whose PM domain is to be set. + * @pd: PM domain to be set, or NULL. + * + * Sets the PM domain the device belongs to. The PM domain of a device needs + * to be set before its probe finishes (it's bound to a driver). + * + * This function must be called with the device lock held. + */ +void dev_pm_domain_set(struct device *dev, struct dev_pm_domain *pd) +{ + if (dev->pm_domain == pd) + return; + + WARN(pd && device_is_bound(dev), + "PM domains can only be changed for unbound devices\n"); + dev->pm_domain = pd; + device_pm_check_callbacks(dev); +} +EXPORT_SYMBOL_GPL(dev_pm_domain_set); + +/** + * dev_pm_domain_set_performance_state - Request a new performance state. + * @dev: The device to make the request for. + * @state: Target performance state for the device. + * + * This function should be called when a new performance state needs to be + * requested for a device that is attached to a PM domain. Note that, the + * support for performance scaling for PM domains is optional. + * + * Returns 0 on success and when performance scaling isn't supported, negative + * error code on failure. + */ +int dev_pm_domain_set_performance_state(struct device *dev, unsigned int state) +{ + if (dev->pm_domain && dev->pm_domain->set_performance_state) + return dev->pm_domain->set_performance_state(dev, state); + + return 0; +} +EXPORT_SYMBOL_GPL(dev_pm_domain_set_performance_state); diff --git a/drivers/base/power/domain.c b/drivers/base/power/domain.c deleted file mode 100644 index bfb8955c406c..000000000000 --- a/drivers/base/power/domain.c +++ /dev/null @@ -1,2179 +0,0 @@ -/* - * drivers/base/power/domain.c - Common code related to device power domains. - * - * Copyright (C) 2011 Rafael J. Wysocki <rjw@sisk.pl>, Renesas Electronics Corp. - * - * This file is released under the GPLv2. - */ - -#include <linux/init.h> -#include <linux/kernel.h> -#include <linux/io.h> -#include <linux/pm_runtime.h> -#include <linux/pm_domain.h> -#include <linux/pm_qos.h> -#include <linux/slab.h> -#include <linux/err.h> -#include <linux/sched.h> -#include <linux/suspend.h> -#include <linux/export.h> - -#define GENPD_DEV_CALLBACK(genpd, type, callback, dev) \ -({ \ - type (*__routine)(struct device *__d); \ - type __ret = (type)0; \ - \ - __routine = genpd->dev_ops.callback; \ - if (__routine) { \ - __ret = __routine(dev); \ - } else { \ - __routine = dev_gpd_data(dev)->ops.callback; \ - if (__routine) \ - __ret = __routine(dev); \ - } \ - __ret; \ -}) - -#define GENPD_DEV_TIMED_CALLBACK(genpd, type, callback, dev, field, name) \ -({ \ - ktime_t __start = ktime_get(); \ - type __retval = GENPD_DEV_CALLBACK(genpd, type, callback, dev); \ - s64 __elapsed = ktime_to_ns(ktime_sub(ktime_get(), __start)); \ - struct gpd_timing_data *__td = &dev_gpd_data(dev)->td; \ - if (!__retval && __elapsed > __td->field) { \ - __td->field = __elapsed; \ - dev_warn(dev, name " latency exceeded, new value %lld ns\n", \ - __elapsed); \ - genpd->max_off_time_changed = true; \ - __td->constraint_changed = true; \ - } \ - __retval; \ -}) - -static LIST_HEAD(gpd_list); -static DEFINE_MUTEX(gpd_list_lock); - -static struct generic_pm_domain *pm_genpd_lookup_name(const char *domain_name) -{ - struct generic_pm_domain *genpd = NULL, *gpd; - - if (IS_ERR_OR_NULL(domain_name)) - return NULL; - - mutex_lock(&gpd_list_lock); - list_for_each_entry(gpd, &gpd_list, gpd_list_node) { - if (!strcmp(gpd->name, domain_name)) { - genpd = gpd; - break; - } - } - mutex_unlock(&gpd_list_lock); - return genpd; -} - -#ifdef CONFIG_PM - -struct generic_pm_domain *dev_to_genpd(struct device *dev) -{ - if (IS_ERR_OR_NULL(dev->pm_domain)) - return ERR_PTR(-EINVAL); - - return pd_to_genpd(dev->pm_domain); -} - -static int genpd_stop_dev(struct generic_pm_domain *genpd, struct device *dev) -{ - return GENPD_DEV_TIMED_CALLBACK(genpd, int, stop, dev, - stop_latency_ns, "stop"); -} - -static int genpd_start_dev(struct generic_pm_domain *genpd, struct device *dev) -{ - return GENPD_DEV_TIMED_CALLBACK(genpd, int, start, dev, - start_latency_ns, "start"); -} - -static bool genpd_sd_counter_dec(struct generic_pm_domain *genpd) -{ - bool ret = false; - - if (!WARN_ON(atomic_read(&genpd->sd_count) == 0)) - ret = !!atomic_dec_and_test(&genpd->sd_count); - - return ret; -} - -static void genpd_sd_counter_inc(struct generic_pm_domain *genpd) -{ - atomic_inc(&genpd->sd_count); - smp_mb__after_atomic_inc(); -} - -static void genpd_acquire_lock(struct generic_pm_domain *genpd) -{ - DEFINE_WAIT(wait); - - mutex_lock(&genpd->lock); - /* - * Wait for the domain to transition into either the active, - * or the power off state. - */ - for (;;) { - prepare_to_wait(&genpd->status_wait_queue, &wait, - TASK_UNINTERRUPTIBLE); - if (genpd->status == GPD_STATE_ACTIVE - || genpd->status == GPD_STATE_POWER_OFF) - break; - mutex_unlock(&genpd->lock); - - schedule(); - - mutex_lock(&genpd->lock); - } - finish_wait(&genpd->status_wait_queue, &wait); -} - -static void genpd_release_lock(struct generic_pm_domain *genpd) -{ - mutex_unlock(&genpd->lock); -} - -static void genpd_set_active(struct generic_pm_domain *genpd) -{ - if (genpd->resume_count == 0) - genpd->status = GPD_STATE_ACTIVE; -} - -static void genpd_recalc_cpu_exit_latency(struct generic_pm_domain *genpd) -{ - s64 usecs64; - - if (!genpd->cpu_data) - return; - - usecs64 = genpd->power_on_latency_ns; - do_div(usecs64, NSEC_PER_USEC); - usecs64 += genpd->cpu_data->saved_exit_latency; - genpd->cpu_data->idle_state->exit_latency = usecs64; -} - -/** - * __pm_genpd_poweron - Restore power to a given PM domain and its masters. - * @genpd: PM domain to power up. - * - * Restore power to @genpd and all of its masters so that it is possible to - * resume a device belonging to it. - */ -static int __pm_genpd_poweron(struct generic_pm_domain *genpd) - __releases(&genpd->lock) __acquires(&genpd->lock) -{ - struct gpd_link *link; - DEFINE_WAIT(wait); - int ret = 0; - - /* If the domain's master is being waited for, we have to wait too. */ - for (;;) { - prepare_to_wait(&genpd->status_wait_queue, &wait, - TASK_UNINTERRUPTIBLE); - if (genpd->status != GPD_STATE_WAIT_MASTER) - break; - mutex_unlock(&genpd->lock); - - schedule(); - - mutex_lock(&genpd->lock); - } - finish_wait(&genpd->status_wait_queue, &wait); - - if (genpd->status == GPD_STATE_ACTIVE - || (genpd->prepared_count > 0 && genpd->suspend_power_off)) - return 0; - - if (genpd->status != GPD_STATE_POWER_OFF) { - genpd_set_active(genpd); - return 0; - } - - if (genpd->cpu_data) { - cpuidle_pause_and_lock(); - genpd->cpu_data->idle_state->disabled = true; - cpuidle_resume_and_unlock(); - goto out; - } - - /* - * The list is guaranteed not to change while the loop below is being - * executed, unless one of the masters' .power_on() callbacks fiddles - * with it. - */ - list_for_each_entry(link, &genpd->slave_links, slave_node) { - genpd_sd_counter_inc(link->master); - genpd->status = GPD_STATE_WAIT_MASTER; - - mutex_unlock(&genpd->lock); - - ret = pm_genpd_poweron(link->master); - - mutex_lock(&genpd->lock); - - /* - * The "wait for parent" status is guaranteed not to change - * while the master is powering on. - */ - genpd->status = GPD_STATE_POWER_OFF; - wake_up_all(&genpd->status_wait_queue); - if (ret) { - genpd_sd_counter_dec(link->master); - goto err; - } - } - - if (genpd->power_on) { - ktime_t time_start = ktime_get(); - s64 elapsed_ns; - - ret = genpd->power_on(genpd); - if (ret) - goto err; - - elapsed_ns = ktime_to_ns(ktime_sub(ktime_get(), time_start)); - if (elapsed_ns > genpd->power_on_latency_ns) { - genpd->power_on_latency_ns = elapsed_ns; - genpd->max_off_time_changed = true; - genpd_recalc_cpu_exit_latency(genpd); - if (genpd->name) - pr_warning("%s: Power-on latency exceeded, " - "new value %lld ns\n", genpd->name, - elapsed_ns); - } - } - - out: - genpd_set_active(genpd); - - return 0; - - err: - list_for_each_entry_continue_reverse(link, &genpd->slave_links, slave_node) - genpd_sd_counter_dec(link->master); - - return ret; -} - -/** - * pm_genpd_poweron - Restore power to a given PM domain and its masters. - * @genpd: PM domain to power up. - */ -int pm_genpd_poweron(struct generic_pm_domain *genpd) -{ - int ret; - - mutex_lock(&genpd->lock); - ret = __pm_genpd_poweron(genpd); - mutex_unlock(&genpd->lock); - return ret; -} - -/** - * pm_genpd_name_poweron - Restore power to a given PM domain and its masters. - * @domain_name: Name of the PM domain to power up. - */ -int pm_genpd_name_poweron(const char *domain_name) -{ - struct generic_pm_domain *genpd; - - genpd = pm_genpd_lookup_name(domain_name); - return genpd ? pm_genpd_poweron(genpd) : -EINVAL; -} - -#endif /* CONFIG_PM */ - -#ifdef CONFIG_PM_RUNTIME - -static int genpd_start_dev_no_timing(struct generic_pm_domain *genpd, - struct device *dev) -{ - return GENPD_DEV_CALLBACK(genpd, int, start, dev); -} - -static int genpd_save_dev(struct generic_pm_domain *genpd, struct device *dev) -{ - return GENPD_DEV_TIMED_CALLBACK(genpd, int, save_state, dev, - save_state_latency_ns, "state save"); -} - -static int genpd_restore_dev(struct generic_pm_domain *genpd, struct device *dev) -{ - return GENPD_DEV_TIMED_CALLBACK(genpd, int, restore_state, dev, - restore_state_latency_ns, - "state restore"); -} - -static int genpd_dev_pm_qos_notifier(struct notifier_block *nb, - unsigned long val, void *ptr) -{ - struct generic_pm_domain_data *gpd_data; - struct device *dev; - - gpd_data = container_of(nb, struct generic_pm_domain_data, nb); - - mutex_lock(&gpd_data->lock); - dev = gpd_data->base.dev; - if (!dev) { - mutex_unlock(&gpd_data->lock); - return NOTIFY_DONE; - } - mutex_unlock(&gpd_data->lock); - - for (;;) { - struct generic_pm_domain *genpd; - struct pm_domain_data *pdd; - - spin_lock_irq(&dev->power.lock); - - pdd = dev->power.subsys_data ? - dev->power.subsys_data->domain_data : NULL; - if (pdd && pdd->dev) { - to_gpd_data(pdd)->td.constraint_changed = true; - genpd = dev_to_genpd(dev); - } else { - genpd = ERR_PTR(-ENODATA); - } - - spin_unlock_irq(&dev->power.lock); - - if (!IS_ERR(genpd)) { - mutex_lock(&genpd->lock); - genpd->max_off_time_changed = true; - mutex_unlock(&genpd->lock); - } - - dev = dev->parent; - if (!dev || dev->power.ignore_children) - break; - } - - return NOTIFY_DONE; -} - -/** - * __pm_genpd_save_device - Save the pre-suspend state of a device. - * @pdd: Domain data of the device to save the state of. - * @genpd: PM domain the device belongs to. - */ -static int __pm_genpd_save_device(struct pm_domain_data *pdd, - struct generic_pm_domain *genpd) - __releases(&genpd->lock) __acquires(&genpd->lock) -{ - struct generic_pm_domain_data *gpd_data = to_gpd_data(pdd); - struct device *dev = pdd->dev; - int ret = 0; - - if (gpd_data->need_restore) - return 0; - - mutex_unlock(&genpd->lock); - - genpd_start_dev(genpd, dev); - ret = genpd_save_dev(genpd, dev); - genpd_stop_dev(genpd, dev); - - mutex_lock(&genpd->lock); - - if (!ret) - gpd_data->need_restore = true; - - return ret; -} - -/** - * __pm_genpd_restore_device - Restore the pre-suspend state of a device. - * @pdd: Domain data of the device to restore the state of. - * @genpd: PM domain the device belongs to. - */ -static void __pm_genpd_restore_device(struct pm_domain_data *pdd, - struct generic_pm_domain *genpd) - __releases(&genpd->lock) __acquires(&genpd->lock) -{ - struct generic_pm_domain_data *gpd_data = to_gpd_data(pdd); - struct device *dev = pdd->dev; - bool need_restore = gpd_data->need_restore; - - gpd_data->need_restore = false; - mutex_unlock(&genpd->lock); - - genpd_start_dev(genpd, dev); - if (need_restore) - genpd_restore_dev(genpd, dev); - - mutex_lock(&genpd->lock); -} - -/** - * genpd_abort_poweroff - Check if a PM domain power off should be aborted. - * @genpd: PM domain to check. - * - * Return true if a PM domain's status changed to GPD_STATE_ACTIVE during - * a "power off" operation, which means that a "power on" has occured in the - * meantime, or if its resume_count field is different from zero, which means - * that one of its devices has been resumed in the meantime. - */ -static bool genpd_abort_poweroff(struct generic_pm_domain *genpd) -{ - return genpd->status == GPD_STATE_WAIT_MASTER - || genpd->status == GPD_STATE_ACTIVE || genpd->resume_count > 0; -} - -/** - * genpd_queue_power_off_work - Queue up the execution of pm_genpd_poweroff(). - * @genpd: PM domait to power off. - * - * Queue up the execution of pm_genpd_poweroff() unless it's already been done - * before. - */ -void genpd_queue_power_off_work(struct generic_pm_domain *genpd) -{ - queue_work(pm_wq, &genpd->power_off_work); -} - -/** - * pm_genpd_poweroff - Remove power from a given PM domain. - * @genpd: PM domain to power down. - * - * If all of the @genpd's devices have been suspended and all of its subdomains - * have been powered down, run the runtime suspend callbacks provided by all of - * the @genpd's devices' drivers and remove power from @genpd. - */ -static int pm_genpd_poweroff(struct generic_pm_domain *genpd) - __releases(&genpd->lock) __acquires(&genpd->lock) -{ - struct pm_domain_data *pdd; - struct gpd_link *link; - unsigned int not_suspended; - int ret = 0; - - start: - /* - * Do not try to power off the domain in the following situations: - * (1) The domain is already in the "power off" state. - * (2) The domain is waiting for its master to power up. - * (3) One of the domain's devices is being resumed right now. - * (4) System suspend is in progress. - */ - if (genpd->status == GPD_STATE_POWER_OFF - || genpd->status == GPD_STATE_WAIT_MASTER - || genpd->resume_count > 0 || genpd->prepared_count > 0) - return 0; - - if (atomic_read(&genpd->sd_count) > 0) - return -EBUSY; - - not_suspended = 0; - list_for_each_entry(pdd, &genpd->dev_list, list_node) { - enum pm_qos_flags_status stat; - - stat = dev_pm_qos_flags(pdd->dev, - PM_QOS_FLAG_NO_POWER_OFF - | PM_QOS_FLAG_REMOTE_WAKEUP); - if (stat > PM_QOS_FLAGS_NONE) - return -EBUSY; - - if (pdd->dev->driver && (!pm_runtime_suspended(pdd->dev) - || pdd->dev->power.irq_safe)) - not_suspended++; - } - - if (not_suspended > genpd->in_progress) - return -EBUSY; - - if (genpd->poweroff_task) { - /* - * Another instance of pm_genpd_poweroff() is executing - * callbacks, so tell it to start over and return. - */ - genpd->status = GPD_STATE_REPEAT; - return 0; - } - - if (genpd->gov && genpd->gov->power_down_ok) { - if (!genpd->gov->power_down_ok(&genpd->domain)) - return -EAGAIN; - } - - genpd->status = GPD_STATE_BUSY; - genpd->poweroff_task = current; - - list_for_each_entry_reverse(pdd, &genpd->dev_list, list_node) { - ret = atomic_read(&genpd->sd_count) == 0 ? - __pm_genpd_save_device(pdd, genpd) : -EBUSY; - - if (genpd_abort_poweroff(genpd)) - goto out; - - if (ret) { - genpd_set_active(genpd); - goto out; - } - - if (genpd->status == GPD_STATE_REPEAT) { - genpd->poweroff_task = NULL; - goto start; - } - } - - if (genpd->cpu_data) { - /* - * If cpu_data is set, cpuidle should turn the domain off when - * the CPU in it is idle. In that case we don't decrement the - * subdomain counts of the master domains, so that power is not - * removed from the current domain prematurely as a result of - * cutting off the masters' power. - */ - genpd->status = GPD_STATE_POWER_OFF; - cpuidle_pause_and_lock(); - genpd->cpu_data->idle_state->disabled = false; - cpuidle_resume_and_unlock(); - goto out; - } - - if (genpd->power_off) { - ktime_t time_start; - s64 elapsed_ns; - - if (atomic_read(&genpd->sd_count) > 0) { - ret = -EBUSY; - goto out; - } - - time_start = ktime_get(); - - /* - * If sd_count > 0 at this point, one of the subdomains hasn't - * managed to call pm_genpd_poweron() for the master yet after - * incrementing it. In that case pm_genpd_poweron() will wait - * for us to drop the lock, so we can call .power_off() and let - * the pm_genpd_poweron() restore power for us (this shouldn't - * happen very often). - */ - ret = genpd->power_off(genpd); - if (ret == -EBUSY) { - genpd_set_active(genpd); - goto out; - } - - elapsed_ns = ktime_to_ns(ktime_sub(ktime_get(), time_start)); - if (elapsed_ns > genpd->power_off_latency_ns) { - genpd->power_off_latency_ns = elapsed_ns; - genpd->max_off_time_changed = true; - if (genpd->name) - pr_warning("%s: Power-off latency exceeded, " - "new value %lld ns\n", genpd->name, - elapsed_ns); - } - } - - genpd->status = GPD_STATE_POWER_OFF; - - list_for_each_entry(link, &genpd->slave_links, slave_node) { - genpd_sd_counter_dec(link->master); - genpd_queue_power_off_work(link->master); - } - - out: - genpd->poweroff_task = NULL; - wake_up_all(&genpd->status_wait_queue); - return ret; -} - -/** - * genpd_power_off_work_fn - Power off PM domain whose subdomain count is 0. - * @work: Work structure used for scheduling the execution of this function. - */ -static void genpd_power_off_work_fn(struct work_struct *work) -{ - struct generic_pm_domain *genpd; - - genpd = container_of(work, struct generic_pm_domain, power_off_work); - - genpd_acquire_lock(genpd); - pm_genpd_poweroff(genpd); - genpd_release_lock(genpd); -} - -/** - * pm_genpd_runtime_suspend - Suspend a device belonging to I/O PM domain. - * @dev: Device to suspend. - * - * Carry out a runtime suspend of a device under the assumption that its - * pm_domain field points to the domain member of an object of type - * struct generic_pm_domain representing a PM domain consisting of I/O devices. - */ -static int pm_genpd_runtime_suspend(struct device *dev) -{ - struct generic_pm_domain *genpd; - bool (*stop_ok)(struct device *__dev); - int ret; - - dev_dbg(dev, "%s()\n", __func__); - - genpd = dev_to_genpd(dev); - if (IS_ERR(genpd)) - return -EINVAL; - - might_sleep_if(!genpd->dev_irq_safe); - - stop_ok = genpd->gov ? genpd->gov->stop_ok : NULL; - if (stop_ok && !stop_ok(dev)) - return -EBUSY; - - ret = genpd_stop_dev(genpd, dev); - if (ret) - return ret; - - /* - * If power.irq_safe is set, this routine will be run with interrupts - * off, so it can't use mutexes. - */ - if (dev->power.irq_safe) - return 0; - - mutex_lock(&genpd->lock); - genpd->in_progress++; - pm_genpd_poweroff(genpd); - genpd->in_progress--; - mutex_unlock(&genpd->lock); - - return 0; -} - -/** - * pm_genpd_runtime_resume - Resume a device belonging to I/O PM domain. - * @dev: Device to resume. - * - * Carry out a runtime resume of a device under the assumption that its - * pm_domain field points to the domain member of an object of type - * struct generic_pm_domain representing a PM domain consisting of I/O devices. - */ -static int pm_genpd_runtime_resume(struct device *dev) -{ - struct generic_pm_domain *genpd; - DEFINE_WAIT(wait); - int ret; - - dev_dbg(dev, "%s()\n", __func__); - - genpd = dev_to_genpd(dev); - if (IS_ERR(genpd)) - return -EINVAL; - - might_sleep_if(!genpd->dev_irq_safe); - - /* If power.irq_safe, the PM domain is never powered off. */ - if (dev->power.irq_safe) - return genpd_start_dev_no_timing(genpd, dev); - - mutex_lock(&genpd->lock); - ret = __pm_genpd_poweron(genpd); - if (ret) { - mutex_unlock(&genpd->lock); - return ret; - } - genpd->status = GPD_STATE_BUSY; - genpd->resume_count++; - for (;;) { - prepare_to_wait(&genpd->status_wait_queue, &wait, - TASK_UNINTERRUPTIBLE); - /* - * If current is the powering off task, we have been called - * reentrantly from one of the device callbacks, so we should - * not wait. - */ - if (!genpd->poweroff_task || genpd->poweroff_task == current) - break; - mutex_unlock(&genpd->lock); - - schedule(); - - mutex_lock(&genpd->lock); - } - finish_wait(&genpd->status_wait_queue, &wait); - __pm_genpd_restore_device(dev->power.subsys_data->domain_data, genpd); - genpd->resume_count--; - genpd_set_active(genpd); - wake_up_all(&genpd->status_wait_queue); - mutex_unlock(&genpd->lock); - - return 0; -} - -/** - * pm_genpd_poweroff_unused - Power off all PM domains with no devices in use. - */ -void pm_genpd_poweroff_unused(void) -{ - struct generic_pm_domain *genpd; - - mutex_lock(&gpd_list_lock); - - list_for_each_entry(genpd, &gpd_list, gpd_list_node) - genpd_queue_power_off_work(genpd); - - mutex_unlock(&gpd_list_lock); -} - -#else - -static inline int genpd_dev_pm_qos_notifier(struct notifier_block *nb, - unsigned long val, void *ptr) -{ - return NOTIFY_DONE; -} - -static inline void genpd_power_off_work_fn(struct work_struct *work) {} - -#define pm_genpd_runtime_suspend NULL -#define pm_genpd_runtime_resume NULL - -#endif /* CONFIG_PM_RUNTIME */ - -#ifdef CONFIG_PM_SLEEP - -/** - * pm_genpd_present - Check if the given PM domain has been initialized. - * @genpd: PM domain to check. - */ -static bool pm_genpd_present(struct generic_pm_domain *genpd) -{ - struct generic_pm_domain *gpd; - - if (IS_ERR_OR_NULL(genpd)) - return false; - - list_for_each_entry(gpd, &gpd_list, gpd_list_node) - if (gpd == genpd) - return true; - - return false; -} - -static bool genpd_dev_active_wakeup(struct generic_pm_domain *genpd, - struct device *dev) -{ - return GENPD_DEV_CALLBACK(genpd, bool, active_wakeup, dev); -} - -static int genpd_suspend_dev(struct generic_pm_domain *genpd, struct device *dev) -{ - return GENPD_DEV_CALLBACK(genpd, int, suspend, dev); -} - -static int genpd_suspend_late(struct generic_pm_domain *genpd, struct device *dev) -{ - return GENPD_DEV_CALLBACK(genpd, int, suspend_late, dev); -} - -static int genpd_resume_early(struct generic_pm_domain *genpd, struct device *dev) -{ - return GENPD_DEV_CALLBACK(genpd, int, resume_early, dev); -} - -static int genpd_resume_dev(struct generic_pm_domain *genpd, struct device *dev) -{ - return GENPD_DEV_CALLBACK(genpd, int, resume, dev); -} - -static int genpd_freeze_dev(struct generic_pm_domain *genpd, struct device *dev) -{ - return GENPD_DEV_CALLBACK(genpd, int, freeze, dev); -} - -static int genpd_freeze_late(struct generic_pm_domain *genpd, struct device *dev) -{ - return GENPD_DEV_CALLBACK(genpd, int, freeze_late, dev); -} - -static int genpd_thaw_early(struct generic_pm_domain *genpd, struct device *dev) -{ - return GENPD_DEV_CALLBACK(genpd, int, thaw_early, dev); -} - -static int genpd_thaw_dev(struct generic_pm_domain *genpd, struct device *dev) -{ - return GENPD_DEV_CALLBACK(genpd, int, thaw, dev); -} - -/** - * pm_genpd_sync_poweroff - Synchronously power off a PM domain and its masters. - * @genpd: PM domain to power off, if possible. - * - * Check if the given PM domain can be powered off (during system suspend or - * hibernation) and do that if so. Also, in that case propagate to its masters. - * - * This function is only called in "noirq" and "syscore" stages of system power - * transitions, so it need not acquire locks (all of the "noirq" callbacks are - * executed sequentially, so it is guaranteed that it will never run twice in - * parallel). - */ -static void pm_genpd_sync_poweroff(struct generic_pm_domain *genpd) -{ - struct gpd_link *link; - - if (genpd->status == GPD_STATE_POWER_OFF) - return; - - if (genpd->suspended_count != genpd->device_count - || atomic_read(&genpd->sd_count) > 0) - return; - - if (genpd->power_off) - genpd->power_off(genpd); - - genpd->status = GPD_STATE_POWER_OFF; - - list_for_each_entry(link, &genpd->slave_links, slave_node) { - genpd_sd_counter_dec(link->master); - pm_genpd_sync_poweroff(link->master); - } -} - -/** - * pm_genpd_sync_poweron - Synchronously power on a PM domain and its masters. - * @genpd: PM domain to power on. - * - * This function is only called in "noirq" and "syscore" stages of system power - * transitions, so it need not acquire locks (all of the "noirq" callbacks are - * executed sequentially, so it is guaranteed that it will never run twice in - * parallel). - */ -static void pm_genpd_sync_poweron(struct generic_pm_domain *genpd) -{ - struct gpd_link *link; - - if (genpd->status != GPD_STATE_POWER_OFF) - return; - - list_for_each_entry(link, &genpd->slave_links, slave_node) { - pm_genpd_sync_poweron(link->master); - genpd_sd_counter_inc(link->master); - } - - if (genpd->power_on) - genpd->power_on(genpd); - - genpd->status = GPD_STATE_ACTIVE; -} - -/** - * resume_needed - Check whether to resume a device before system suspend. - * @dev: Device to check. - * @genpd: PM domain the device belongs to. - * - * There are two cases in which a device that can wake up the system from sleep - * states should be resumed by pm_genpd_prepare(): (1) if the device is enabled - * to wake up the system and it has to remain active for this purpose while the - * system is in the sleep state and (2) if the device is not enabled to wake up - * the system from sleep states and it generally doesn't generate wakeup signals - * by itself (those signals are generated on its behalf by other parts of the - * system). In the latter case it may be necessary to reconfigure the device's - * wakeup settings during system suspend, because it may have been set up to - * signal remote wakeup from the system's working state as needed by runtime PM. - * Return 'true' in either of the above cases. - */ -static bool resume_needed(struct device *dev, struct generic_pm_domain *genpd) -{ - bool active_wakeup; - - if (!device_can_wakeup(dev)) - return false; - - active_wakeup = genpd_dev_active_wakeup(genpd, dev); - return device_may_wakeup(dev) ? active_wakeup : !active_wakeup; -} - -/** - * pm_genpd_prepare - Start power transition of a device in a PM domain. - * @dev: Device to start the transition of. - * - * Start a power transition of a device (during a system-wide power transition) - * under the assumption that its pm_domain field points to the domain member of - * an object of type struct generic_pm_domain representing a PM domain - * consisting of I/O devices. - */ -static int pm_genpd_prepare(struct device *dev) -{ - struct generic_pm_domain *genpd; - int ret; - - dev_dbg(dev, "%s()\n", __func__); - - genpd = dev_to_genpd(dev); - if (IS_ERR(genpd)) - return -EINVAL; - - /* - * If a wakeup request is pending for the device, it should be woken up - * at this point and a system wakeup event should be reported if it's - * set up to wake up the system from sleep states. - */ - pm_runtime_get_noresume(dev); - if (pm_runtime_barrier(dev) && device_may_wakeup(dev)) - pm_wakeup_event(dev, 0); - - if (pm_wakeup_pending()) { - pm_runtime_put(dev); - return -EBUSY; - } - - if (resume_needed(dev, genpd)) - pm_runtime_resume(dev); - - genpd_acquire_lock(genpd); - - if (genpd->prepared_count++ == 0) { - genpd->suspended_count = 0; - genpd->suspend_power_off = genpd->status == GPD_STATE_POWER_OFF; - } - - genpd_release_lock(genpd); - - if (genpd->suspend_power_off) { - pm_runtime_put_noidle(dev); - return 0; - } - - /* - * The PM domain must be in the GPD_STATE_ACTIVE state at this point, - * so pm_genpd_poweron() will return immediately, but if the device - * is suspended (e.g. it's been stopped by genpd_stop_dev()), we need - * to make it operational. - */ - pm_runtime_resume(dev); - __pm_runtime_disable(dev, false); - - ret = pm_generic_prepare(dev); - if (ret) { - mutex_lock(&genpd->lock); - - if (--genpd->prepared_count == 0) - genpd->suspend_power_off = false; - - mutex_unlock(&genpd->lock); - pm_runtime_enable(dev); - } - - pm_runtime_put(dev); - return ret; -} - -/** - * pm_genpd_suspend - Suspend a device belonging to an I/O PM domain. - * @dev: Device to suspend. - * - * Suspend a device under the assumption that its pm_domain field points to the - * domain member of an object of type struct generic_pm_domain representing - * a PM domain consisting of I/O devices. - */ -static int pm_genpd_suspend(struct device *dev) -{ - struct generic_pm_domain *genpd; - - dev_dbg(dev, "%s()\n", __func__); - - genpd = dev_to_genpd(dev); - if (IS_ERR(genpd)) - return -EINVAL; - - return genpd->suspend_power_off ? 0 : genpd_suspend_dev(genpd, dev); -} - -/** - * pm_genpd_suspend_late - Late suspend of a device from an I/O PM domain. - * @dev: Device to suspend. - * - * Carry out a late suspend of a device under the assumption that its - * pm_domain field points to the domain member of an object of type - * struct generic_pm_domain representing a PM domain consisting of I/O devices. - */ -static int pm_genpd_suspend_late(struct device *dev) -{ - struct generic_pm_domain *genpd; - - dev_dbg(dev, "%s()\n", __func__); - - genpd = dev_to_genpd(dev); - if (IS_ERR(genpd)) - return -EINVAL; - - return genpd->suspend_power_off ? 0 : genpd_suspend_late(genpd, dev); -} - -/** - * pm_genpd_suspend_noirq - Completion of suspend of device in an I/O PM domain. - * @dev: Device to suspend. - * - * Stop the device and remove power from the domain if all devices in it have - * been stopped. - */ -static int pm_genpd_suspend_noirq(struct device *dev) -{ - struct generic_pm_domain *genpd; - - dev_dbg(dev, "%s()\n", __func__); - - genpd = dev_to_genpd(dev); - if (IS_ERR(genpd)) - return -EINVAL; - - if (genpd->suspend_power_off - || (dev->power.wakeup_path && genpd_dev_active_wakeup(genpd, dev))) - return 0; - - genpd_stop_dev(genpd, dev); - - /* - * Since all of the "noirq" callbacks are executed sequentially, it is - * guaranteed that this function will never run twice in parallel for - * the same PM domain, so it is not necessary to use locking here. - */ - genpd->suspended_count++; - pm_genpd_sync_poweroff(genpd); - - return 0; -} - -/** - * pm_genpd_resume_noirq - Start of resume of device in an I/O PM domain. - * @dev: Device to resume. - * - * Restore power to the device's PM domain, if necessary, and start the device. - */ -static int pm_genpd_resume_noirq(struct device *dev) -{ - struct generic_pm_domain *genpd; - - dev_dbg(dev, "%s()\n", __func__); - - genpd = dev_to_genpd(dev); - if (IS_ERR(genpd)) - return -EINVAL; - - if (genpd->suspend_power_off - || (dev->power.wakeup_path && genpd_dev_active_wakeup(genpd, dev))) - return 0; - - /* - * Since all of the "noirq" callbacks are executed sequentially, it is - * guaranteed that this function will never run twice in parallel for - * the same PM domain, so it is not necessary to use locking here. - */ - pm_genpd_sync_poweron(genpd); - genpd->suspended_count--; - - return genpd_start_dev(genpd, dev); -} - -/** - * pm_genpd_resume_early - Early resume of a device in an I/O PM domain. - * @dev: Device to resume. - * - * Carry out an early resume of a device under the assumption that its - * pm_domain field points to the domain member of an object of type - * struct generic_pm_domain representing a power domain consisting of I/O - * devices. - */ -static int pm_genpd_resume_early(struct device *dev) -{ - struct generic_pm_domain *genpd; - - dev_dbg(dev, "%s()\n", __func__); - - genpd = dev_to_genpd(dev); - if (IS_ERR(genpd)) - return -EINVAL; - - return genpd->suspend_power_off ? 0 : genpd_resume_early(genpd, dev); -} - -/** - * pm_genpd_resume - Resume of device in an I/O PM domain. - * @dev: Device to resume. - * - * Resume a device under the assumption that its pm_domain field points to the - * domain member of an object of type struct generic_pm_domain representing - * a power domain consisting of I/O devices. - */ -static int pm_genpd_resume(struct device *dev) -{ - struct generic_pm_domain *genpd; - - dev_dbg(dev, "%s()\n", __func__); - - genpd = dev_to_genpd(dev); - if (IS_ERR(genpd)) - return -EINVAL; - - return genpd->suspend_power_off ? 0 : genpd_resume_dev(genpd, dev); -} - -/** - * pm_genpd_freeze - Freezing a device in an I/O PM domain. - * @dev: Device to freeze. - * - * Freeze a device under the assumption that its pm_domain field points to the - * domain member of an object of type struct generic_pm_domain representing - * a power domain consisting of I/O devices. - */ -static int pm_genpd_freeze(struct device *dev) -{ - struct generic_pm_domain *genpd; - - dev_dbg(dev, "%s()\n", __func__); - - genpd = dev_to_genpd(dev); - if (IS_ERR(genpd)) - return -EINVAL; - - return genpd->suspend_power_off ? 0 : genpd_freeze_dev(genpd, dev); -} - -/** - * pm_genpd_freeze_late - Late freeze of a device in an I/O PM domain. - * @dev: Device to freeze. - * - * Carry out a late freeze of a device under the assumption that its - * pm_domain field points to the domain member of an object of type - * struct generic_pm_domain representing a power domain consisting of I/O - * devices. - */ -static int pm_genpd_freeze_late(struct device *dev) -{ - struct generic_pm_domain *genpd; - - dev_dbg(dev, "%s()\n", __func__); - - genpd = dev_to_genpd(dev); - if (IS_ERR(genpd)) - return -EINVAL; - - return genpd->suspend_power_off ? 0 : genpd_freeze_late(genpd, dev); -} - -/** - * pm_genpd_freeze_noirq - Completion of freezing a device in an I/O PM domain. - * @dev: Device to freeze. - * - * Carry out a late freeze of a device under the assumption that its - * pm_domain field points to the domain member of an object of type - * struct generic_pm_domain representing a power domain consisting of I/O - * devices. - */ -static int pm_genpd_freeze_noirq(struct device *dev) -{ - struct generic_pm_domain *genpd; - - dev_dbg(dev, "%s()\n", __func__); - - genpd = dev_to_genpd(dev); - if (IS_ERR(genpd)) - return -EINVAL; - - return genpd->suspend_power_off ? 0 : genpd_stop_dev(genpd, dev); -} - -/** - * pm_genpd_thaw_noirq - Early thaw of device in an I/O PM domain. - * @dev: Device to thaw. - * - * Start the device, unless power has been removed from the domain already - * before the system transition. - */ -static int pm_genpd_thaw_noirq(struct device *dev) -{ - struct generic_pm_domain *genpd; - - dev_dbg(dev, "%s()\n", __func__); - - genpd = dev_to_genpd(dev); - if (IS_ERR(genpd)) - return -EINVAL; - - return genpd->suspend_power_off ? 0 : genpd_start_dev(genpd, dev); -} - -/** - * pm_genpd_thaw_early - Early thaw of device in an I/O PM domain. - * @dev: Device to thaw. - * - * Carry out an early thaw of a device under the assumption that its - * pm_domain field points to the domain member of an object of type - * struct generic_pm_domain representing a power domain consisting of I/O - * devices. - */ -static int pm_genpd_thaw_early(struct device *dev) -{ - struct generic_pm_domain *genpd; - - dev_dbg(dev, "%s()\n", __func__); - - genpd = dev_to_genpd(dev); - if (IS_ERR(genpd)) - return -EINVAL; - - return genpd->suspend_power_off ? 0 : genpd_thaw_early(genpd, dev); -} - -/** - * pm_genpd_thaw - Thaw a device belonging to an I/O power domain. - * @dev: Device to thaw. - * - * Thaw a device under the assumption that its pm_domain field points to the - * domain member of an object of type struct generic_pm_domain representing - * a power domain consisting of I/O devices. - */ -static int pm_genpd_thaw(struct device *dev) -{ - struct generic_pm_domain *genpd; - - dev_dbg(dev, "%s()\n", __func__); - - genpd = dev_to_genpd(dev); - if (IS_ERR(genpd)) - return -EINVAL; - - return genpd->suspend_power_off ? 0 : genpd_thaw_dev(genpd, dev); -} - -/** - * pm_genpd_restore_noirq - Start of restore of device in an I/O PM domain. - * @dev: Device to resume. - * - * Make sure the domain will be in the same power state as before the - * hibernation the system is resuming from and start the device if necessary. - */ -static int pm_genpd_restore_noirq(struct device *dev) -{ - struct generic_pm_domain *genpd; - - dev_dbg(dev, "%s()\n", __func__); - - genpd = dev_to_genpd(dev); - if (IS_ERR(genpd)) - return -EINVAL; - - /* - * Since all of the "noirq" callbacks are executed sequentially, it is - * guaranteed that this function will never run twice in parallel for - * the same PM domain, so it is not necessary to use locking here. - * - * At this point suspended_count == 0 means we are being run for the - * first time for the given domain in the present cycle. - */ - if (genpd->suspended_count++ == 0) { - /* - * The boot kernel might put the domain into arbitrary state, - * so make it appear as powered off to pm_genpd_sync_poweron(), - * so that it tries to power it on in case it was really off. - */ - genpd->status = GPD_STATE_POWER_OFF; - if (genpd->suspend_power_off) { - /* - * If the domain was off before the hibernation, make - * sure it will be off going forward. - */ - if (genpd->power_off) - genpd->power_off(genpd); - - return 0; - } - } - - if (genpd->suspend_power_off) - return 0; - - pm_genpd_sync_poweron(genpd); - - return genpd_start_dev(genpd, dev); -} - -/** - * pm_genpd_complete - Complete power transition of a device in a power domain. - * @dev: Device to complete the transition of. - * - * Complete a power transition of a device (during a system-wide power - * transition) under the assumption that its pm_domain field points to the - * domain member of an object of type struct generic_pm_domain representing - * a power domain consisting of I/O devices. - */ -static void pm_genpd_complete(struct device *dev) -{ - struct generic_pm_domain *genpd; - bool run_complete; - - dev_dbg(dev, "%s()\n", __func__); - - genpd = dev_to_genpd(dev); - if (IS_ERR(genpd)) - return; - - mutex_lock(&genpd->lock); - - run_complete = !genpd->suspend_power_off; - if (--genpd->prepared_count == 0) - genpd->suspend_power_off = false; - - mutex_unlock(&genpd->lock); - - if (run_complete) { - pm_generic_complete(dev); - pm_runtime_set_active(dev); - pm_runtime_enable(dev); - pm_request_idle(dev); - } -} - -/** - * pm_genpd_syscore_switch - Switch power during system core suspend or resume. - * @dev: Device that normally is marked as "always on" to switch power for. - * - * This routine may only be called during the system core (syscore) suspend or - * resume phase for devices whose "always on" flags are set. - */ -void pm_genpd_syscore_switch(struct device *dev, bool suspend) -{ - struct generic_pm_domain *genpd; - - genpd = dev_to_genpd(dev); - if (!pm_genpd_present(genpd)) - return; - - if (suspend) { - genpd->suspended_count++; - pm_genpd_sync_poweroff(genpd); - } else { - pm_genpd_sync_poweron(genpd); - genpd->suspended_count--; - } -} -EXPORT_SYMBOL_GPL(pm_genpd_syscore_switch); - -#else - -#define pm_genpd_prepare NULL -#define pm_genpd_suspend NULL -#define pm_genpd_suspend_late NULL -#define pm_genpd_suspend_noirq NULL -#define pm_genpd_resume_early NULL -#define pm_genpd_resume_noirq NULL -#define pm_genpd_resume NULL -#define pm_genpd_freeze NULL -#define pm_genpd_freeze_late NULL -#define pm_genpd_freeze_noirq NULL -#define pm_genpd_thaw_early NULL -#define pm_genpd_thaw_noirq NULL -#define pm_genpd_thaw NULL -#define pm_genpd_restore_noirq NULL -#define pm_genpd_complete NULL - -#endif /* CONFIG_PM_SLEEP */ - -static struct generic_pm_domain_data *__pm_genpd_alloc_dev_data(struct device *dev) -{ - struct generic_pm_domain_data *gpd_data; - - gpd_data = kzalloc(sizeof(*gpd_data), GFP_KERNEL); - if (!gpd_data) - return NULL; - - mutex_init(&gpd_data->lock); - gpd_data->nb.notifier_call = genpd_dev_pm_qos_notifier; - dev_pm_qos_add_notifier(dev, &gpd_data->nb); - return gpd_data; -} - -static void __pm_genpd_free_dev_data(struct device *dev, - struct generic_pm_domain_data *gpd_data) -{ - dev_pm_qos_remove_notifier(dev, &gpd_data->nb); - kfree(gpd_data); -} - -/** - * __pm_genpd_add_device - Add a device to an I/O PM domain. - * @genpd: PM domain to add the device to. - * @dev: Device to be added. - * @td: Set of PM QoS timing parameters to attach to the device. - */ -int __pm_genpd_add_device(struct generic_pm_domain *genpd, struct device *dev, - struct gpd_timing_data *td) -{ - struct generic_pm_domain_data *gpd_data_new, *gpd_data = NULL; - struct pm_domain_data *pdd; - int ret = 0; - - dev_dbg(dev, "%s()\n", __func__); - - if (IS_ERR_OR_NULL(genpd) || IS_ERR_OR_NULL(dev)) - return -EINVAL; - - gpd_data_new = __pm_genpd_alloc_dev_data(dev); - if (!gpd_data_new) - return -ENOMEM; - - genpd_acquire_lock(genpd); - - if (genpd->prepared_count > 0) { - ret = -EAGAIN; - goto out; - } - - list_for_each_entry(pdd, &genpd->dev_list, list_node) - if (pdd->dev == dev) { - ret = -EINVAL; - goto out; - } - - ret = dev_pm_get_subsys_data(dev); - if (ret) - goto out; - - genpd->device_count++; - genpd->max_off_time_changed = true; - - spin_lock_irq(&dev->power.lock); - - dev->pm_domain = &genpd->domain; - if (dev->power.subsys_data->domain_data) { - gpd_data = to_gpd_data(dev->power.subsys_data->domain_data); - } else { - gpd_data = gpd_data_new; - dev->power.subsys_data->domain_data = &gpd_data->base; - } - gpd_data->refcount++; - if (td) - gpd_data->td = *td; - - spin_unlock_irq(&dev->power.lock); - - mutex_lock(&gpd_data->lock); - gpd_data->base.dev = dev; - list_add_tail(&gpd_data->base.list_node, &genpd->dev_list); - gpd_data->need_restore = genpd->status == GPD_STATE_POWER_OFF; - gpd_data->td.constraint_changed = true; - gpd_data->td.effective_constraint_ns = -1; - mutex_unlock(&gpd_data->lock); - - out: - genpd_release_lock(genpd); - - if (gpd_data != gpd_data_new) - __pm_genpd_free_dev_data(dev, gpd_data_new); - - return ret; -} - -/** - * __pm_genpd_of_add_device - Add a device to an I/O PM domain. - * @genpd_node: Device tree node pointer representing a PM domain to which the - * the device is added to. - * @dev: Device to be added. - * @td: Set of PM QoS timing parameters to attach to the device. - */ -int __pm_genpd_of_add_device(struct device_node *genpd_node, struct device *dev, - struct gpd_timing_data *td) -{ - struct generic_pm_domain *genpd = NULL, *gpd; - - dev_dbg(dev, "%s()\n", __func__); - - if (IS_ERR_OR_NULL(genpd_node) || IS_ERR_OR_NULL(dev)) - return -EINVAL; - - mutex_lock(&gpd_list_lock); - list_for_each_entry(gpd, &gpd_list, gpd_list_node) { - if (gpd->of_node == genpd_node) { - genpd = gpd; - break; - } - } - mutex_unlock(&gpd_list_lock); - - if (!genpd) - return -EINVAL; - - return __pm_genpd_add_device(genpd, dev, td); -} - - -/** - * __pm_genpd_name_add_device - Find I/O PM domain and add a device to it. - * @domain_name: Name of the PM domain to add the device to. - * @dev: Device to be added. - * @td: Set of PM QoS timing parameters to attach to the device. - */ -int __pm_genpd_name_add_device(const char *domain_name, struct device *dev, - struct gpd_timing_data *td) -{ - return __pm_genpd_add_device(pm_genpd_lookup_name(domain_name), dev, td); -} - -/** - * pm_genpd_remove_device - Remove a device from an I/O PM domain. - * @genpd: PM domain to remove the device from. - * @dev: Device to be removed. - */ -int pm_genpd_remove_device(struct generic_pm_domain *genpd, - struct device *dev) -{ - struct generic_pm_domain_data *gpd_data; - struct pm_domain_data *pdd; - bool remove = false; - int ret = 0; - - dev_dbg(dev, "%s()\n", __func__); - - if (IS_ERR_OR_NULL(genpd) || IS_ERR_OR_NULL(dev) - || IS_ERR_OR_NULL(dev->pm_domain) - || pd_to_genpd(dev->pm_domain) != genpd) - return -EINVAL; - - genpd_acquire_lock(genpd); - - if (genpd->prepared_count > 0) { - ret = -EAGAIN; - goto out; - } - - genpd->device_count--; - genpd->max_off_time_changed = true; - - spin_lock_irq(&dev->power.lock); - - dev->pm_domain = NULL; - pdd = dev->power.subsys_data->domain_data; - list_del_init(&pdd->list_node); - gpd_data = to_gpd_data(pdd); - if (--gpd_data->refcount == 0) { - dev->power.subsys_data->domain_data = NULL; - remove = true; - } - - spin_unlock_irq(&dev->power.lock); - - mutex_lock(&gpd_data->lock); - pdd->dev = NULL; - mutex_unlock(&gpd_data->lock); - - genpd_release_lock(genpd); - - dev_pm_put_subsys_data(dev); - if (remove) - __pm_genpd_free_dev_data(dev, gpd_data); - - return 0; - - out: - genpd_release_lock(genpd); - - return ret; -} - -/** - * pm_genpd_dev_need_restore - Set/unset the device's "need restore" flag. - * @dev: Device to set/unset the flag for. - * @val: The new value of the device's "need restore" flag. - */ -void pm_genpd_dev_need_restore(struct device *dev, bool val) -{ - struct pm_subsys_data *psd; - unsigned long flags; - - spin_lock_irqsave(&dev->power.lock, flags); - - psd = dev_to_psd(dev); - if (psd && psd->domain_data) - to_gpd_data(psd->domain_data)->need_restore = val; - - spin_unlock_irqrestore(&dev->power.lock, flags); -} -EXPORT_SYMBOL_GPL(pm_genpd_dev_need_restore); - -/** - * pm_genpd_add_subdomain - Add a subdomain to an I/O PM domain. - * @genpd: Master PM domain to add the subdomain to. - * @subdomain: Subdomain to be added. - */ -int pm_genpd_add_subdomain(struct generic_pm_domain *genpd, - struct generic_pm_domain *subdomain) -{ - struct gpd_link *link; - int ret = 0; - - if (IS_ERR_OR_NULL(genpd) || IS_ERR_OR_NULL(subdomain) - || genpd == subdomain) - return -EINVAL; - - start: - genpd_acquire_lock(genpd); - mutex_lock_nested(&subdomain->lock, SINGLE_DEPTH_NESTING); - - if (subdomain->status != GPD_STATE_POWER_OFF - && subdomain->status != GPD_STATE_ACTIVE) { - mutex_unlock(&subdomain->lock); - genpd_release_lock(genpd); - goto start; - } - - if (genpd->status == GPD_STATE_POWER_OFF - && subdomain->status != GPD_STATE_POWER_OFF) { - ret = -EINVAL; - goto out; - } - - list_for_each_entry(link, &genpd->master_links, master_node) { - if (link->slave == subdomain && link->master == genpd) { - ret = -EINVAL; - goto out; - } - } - - link = kzalloc(sizeof(*link), GFP_KERNEL); - if (!link) { - ret = -ENOMEM; - goto out; - } - link->master = genpd; - list_add_tail(&link->master_node, &genpd->master_links); - link->slave = subdomain; - list_add_tail(&link->slave_node, &subdomain->slave_links); - if (subdomain->status != GPD_STATE_POWER_OFF) - genpd_sd_counter_inc(genpd); - - out: - mutex_unlock(&subdomain->lock); - genpd_release_lock(genpd); - - return ret; -} - -/** - * pm_genpd_add_subdomain_names - Add a subdomain to an I/O PM domain. - * @master_name: Name of the master PM domain to add the subdomain to. - * @subdomain_name: Name of the subdomain to be added. - */ -int pm_genpd_add_subdomain_names(const char *master_name, - const char *subdomain_name) -{ - struct generic_pm_domain *master = NULL, *subdomain = NULL, *gpd; - - if (IS_ERR_OR_NULL(master_name) || IS_ERR_OR_NULL(subdomain_name)) - return -EINVAL; - - mutex_lock(&gpd_list_lock); - list_for_each_entry(gpd, &gpd_list, gpd_list_node) { - if (!master && !strcmp(gpd->name, master_name)) - master = gpd; - - if (!subdomain && !strcmp(gpd->name, subdomain_name)) - subdomain = gpd; - - if (master && subdomain) - break; - } - mutex_unlock(&gpd_list_lock); - - return pm_genpd_add_subdomain(master, subdomain); -} - -/** - * pm_genpd_remove_subdomain - Remove a subdomain from an I/O PM domain. - * @genpd: Master PM domain to remove the subdomain from. - * @subdomain: Subdomain to be removed. - */ -int pm_genpd_remove_subdomain(struct generic_pm_domain *genpd, - struct generic_pm_domain *subdomain) -{ - struct gpd_link *link; - int ret = -EINVAL; - - if (IS_ERR_OR_NULL(genpd) || IS_ERR_OR_NULL(subdomain)) - return -EINVAL; - - start: - genpd_acquire_lock(genpd); - - list_for_each_entry(link, &genpd->master_links, master_node) { - if (link->slave != subdomain) - continue; - - mutex_lock_nested(&subdomain->lock, SINGLE_DEPTH_NESTING); - - if (subdomain->status != GPD_STATE_POWER_OFF - && subdomain->status != GPD_STATE_ACTIVE) { - mutex_unlock(&subdomain->lock); - genpd_release_lock(genpd); - goto start; - } - - list_del(&link->master_node); - list_del(&link->slave_node); - kfree(link); - if (subdomain->status != GPD_STATE_POWER_OFF) - genpd_sd_counter_dec(genpd); - - mutex_unlock(&subdomain->lock); - - ret = 0; - break; - } - - genpd_release_lock(genpd); - - return ret; -} - -/** - * pm_genpd_add_callbacks - Add PM domain callbacks to a given device. - * @dev: Device to add the callbacks to. - * @ops: Set of callbacks to add. - * @td: Timing data to add to the device along with the callbacks (optional). - * - * Every call to this routine should be balanced with a call to - * __pm_genpd_remove_callbacks() and they must not be nested. - */ -int pm_genpd_add_callbacks(struct device *dev, struct gpd_dev_ops *ops, - struct gpd_timing_data *td) -{ - struct generic_pm_domain_data *gpd_data_new, *gpd_data = NULL; - int ret = 0; - - if (!(dev && ops)) - return -EINVAL; - - gpd_data_new = __pm_genpd_alloc_dev_data(dev); - if (!gpd_data_new) - return -ENOMEM; - - pm_runtime_disable(dev); - device_pm_lock(); - - ret = dev_pm_get_subsys_data(dev); - if (ret) - goto out; - - spin_lock_irq(&dev->power.lock); - - if (dev->power.subsys_data->domain_data) { - gpd_data = to_gpd_data(dev->power.subsys_data->domain_data); - } else { - gpd_data = gpd_data_new; - dev->power.subsys_data->domain_data = &gpd_data->base; - } - gpd_data->refcount++; - gpd_data->ops = *ops; - if (td) - gpd_data->td = *td; - - spin_unlock_irq(&dev->power.lock); - - out: - device_pm_unlock(); - pm_runtime_enable(dev); - - if (gpd_data != gpd_data_new) - __pm_genpd_free_dev_data(dev, gpd_data_new); - - return ret; -} -EXPORT_SYMBOL_GPL(pm_genpd_add_callbacks); - -/** - * __pm_genpd_remove_callbacks - Remove PM domain callbacks from a given device. - * @dev: Device to remove the callbacks from. - * @clear_td: If set, clear the device's timing data too. - * - * This routine can only be called after pm_genpd_add_callbacks(). - */ -int __pm_genpd_remove_callbacks(struct device *dev, bool clear_td) -{ - struct generic_pm_domain_data *gpd_data = NULL; - bool remove = false; - int ret = 0; - - if (!(dev && dev->power.subsys_data)) - return -EINVAL; - - pm_runtime_disable(dev); - device_pm_lock(); - - spin_lock_irq(&dev->power.lock); - - if (dev->power.subsys_data->domain_data) { - gpd_data = to_gpd_data(dev->power.subsys_data->domain_data); - gpd_data->ops = (struct gpd_dev_ops){ NULL }; - if (clear_td) - gpd_data->td = (struct gpd_timing_data){ 0 }; - - if (--gpd_data->refcount == 0) { - dev->power.subsys_data->domain_data = NULL; - remove = true; - } - } else { - ret = -EINVAL; - } - - spin_unlock_irq(&dev->power.lock); - - device_pm_unlock(); - pm_runtime_enable(dev); - - if (ret) - return ret; - - dev_pm_put_subsys_data(dev); - if (remove) - __pm_genpd_free_dev_data(dev, gpd_data); - - return 0; -} -EXPORT_SYMBOL_GPL(__pm_genpd_remove_callbacks); - -/** - * pm_genpd_attach_cpuidle - Connect the given PM domain with cpuidle. - * @genpd: PM domain to be connected with cpuidle. - * @state: cpuidle state this domain can disable/enable. - * - * Make a PM domain behave as though it contained a CPU core, that is, instead - * of calling its power down routine it will enable the given cpuidle state so - * that the cpuidle subsystem can power it down (if possible and desirable). - */ -int pm_genpd_attach_cpuidle(struct generic_pm_domain *genpd, int state) -{ - struct cpuidle_driver *cpuidle_drv; - struct gpd_cpu_data *cpu_data; - struct cpuidle_state *idle_state; - int ret = 0; - - if (IS_ERR_OR_NULL(genpd) || state < 0) - return -EINVAL; - - genpd_acquire_lock(genpd); - - if (genpd->cpu_data) { - ret = -EEXIST; - goto out; - } - cpu_data = kzalloc(sizeof(*cpu_data), GFP_KERNEL); - if (!cpu_data) { - ret = -ENOMEM; - goto out; - } - cpuidle_drv = cpuidle_driver_ref(); - if (!cpuidle_drv) { - ret = -ENODEV; - goto err_drv; - } - if (cpuidle_drv->state_count <= state) { - ret = -EINVAL; - goto err; - } - idle_state = &cpuidle_drv->states[state]; - if (!idle_state->disabled) { - ret = -EAGAIN; - goto err; - } - cpu_data->idle_state = idle_state; - cpu_data->saved_exit_latency = idle_state->exit_latency; - genpd->cpu_data = cpu_data; - genpd_recalc_cpu_exit_latency(genpd); - - out: - genpd_release_lock(genpd); - return ret; - - err: - cpuidle_driver_unref(); - - err_drv: - kfree(cpu_data); - goto out; -} - -/** - * pm_genpd_name_attach_cpuidle - Find PM domain and connect cpuidle to it. - * @name: Name of the domain to connect to cpuidle. - * @state: cpuidle state this domain can manipulate. - */ -int pm_genpd_name_attach_cpuidle(const char *name, int state) -{ - return pm_genpd_attach_cpuidle(pm_genpd_lookup_name(name), state); -} - -/** - * pm_genpd_detach_cpuidle - Remove the cpuidle connection from a PM domain. - * @genpd: PM domain to remove the cpuidle connection from. - * - * Remove the cpuidle connection set up by pm_genpd_attach_cpuidle() from the - * given PM domain. - */ -int pm_genpd_detach_cpuidle(struct generic_pm_domain *genpd) -{ - struct gpd_cpu_data *cpu_data; - struct cpuidle_state *idle_state; - int ret = 0; - - if (IS_ERR_OR_NULL(genpd)) - return -EINVAL; - - genpd_acquire_lock(genpd); - - cpu_data = genpd->cpu_data; - if (!cpu_data) { - ret = -ENODEV; - goto out; - } - idle_state = cpu_data->idle_state; - if (!idle_state->disabled) { - ret = -EAGAIN; - goto out; - } - idle_state->exit_latency = cpu_data->saved_exit_latency; - cpuidle_driver_unref(); - genpd->cpu_data = NULL; - kfree(cpu_data); - - out: - genpd_release_lock(genpd); - return ret; -} - -/** - * pm_genpd_name_detach_cpuidle - Find PM domain and disconnect cpuidle from it. - * @name: Name of the domain to disconnect cpuidle from. - */ -int pm_genpd_name_detach_cpuidle(const char *name) -{ - return pm_genpd_detach_cpuidle(pm_genpd_lookup_name(name)); -} - -/* Default device callbacks for generic PM domains. */ - -/** - * pm_genpd_default_save_state - Default "save device state" for PM domians. - * @dev: Device to handle. - */ -static int pm_genpd_default_save_state(struct device *dev) -{ - int (*cb)(struct device *__dev); - - cb = dev_gpd_data(dev)->ops.save_state; - if (cb) - return cb(dev); - - if (dev->type && dev->type->pm) - cb = dev->type->pm->runtime_suspend; - else if (dev->class && dev->class->pm) - cb = dev->class->pm->runtime_suspend; - else if (dev->bus && dev->bus->pm) - cb = dev->bus->pm->runtime_suspend; - else - cb = NULL; - - if (!cb && dev->driver && dev->driver->pm) - cb = dev->driver->pm->runtime_suspend; - - return cb ? cb(dev) : 0; -} - -/** - * pm_genpd_default_restore_state - Default PM domians "restore device state". - * @dev: Device to handle. - */ -static int pm_genpd_default_restore_state(struct device *dev) -{ - int (*cb)(struct device *__dev); - - cb = dev_gpd_data(dev)->ops.restore_state; - if (cb) - return cb(dev); - - if (dev->type && dev->type->pm) - cb = dev->type->pm->runtime_resume; - else if (dev->class && dev->class->pm) - cb = dev->class->pm->runtime_resume; - else if (dev->bus && dev->bus->pm) - cb = dev->bus->pm->runtime_resume; - else - cb = NULL; - - if (!cb && dev->driver && dev->driver->pm) - cb = dev->driver->pm->runtime_resume; - - return cb ? cb(dev) : 0; -} - -#ifdef CONFIG_PM_SLEEP - -/** - * pm_genpd_default_suspend - Default "device suspend" for PM domians. - * @dev: Device to handle. - */ -static int pm_genpd_default_suspend(struct device *dev) -{ - int (*cb)(struct device *__dev) = dev_gpd_data(dev)->ops.suspend; - - return cb ? cb(dev) : pm_generic_suspend(dev); -} - -/** - * pm_genpd_default_suspend_late - Default "late device suspend" for PM domians. - * @dev: Device to handle. - */ -static int pm_genpd_default_suspend_late(struct device *dev) -{ - int (*cb)(struct device *__dev) = dev_gpd_data(dev)->ops.suspend_late; - - return cb ? cb(dev) : pm_generic_suspend_late(dev); -} - -/** - * pm_genpd_default_resume_early - Default "early device resume" for PM domians. - * @dev: Device to handle. - */ -static int pm_genpd_default_resume_early(struct device *dev) -{ - int (*cb)(struct device *__dev) = dev_gpd_data(dev)->ops.resume_early; - - return cb ? cb(dev) : pm_generic_resume_early(dev); -} - -/** - * pm_genpd_default_resume - Default "device resume" for PM domians. - * @dev: Device to handle. - */ -static int pm_genpd_default_resume(struct device *dev) -{ - int (*cb)(struct device *__dev) = dev_gpd_data(dev)->ops.resume; - - return cb ? cb(dev) : pm_generic_resume(dev); -} - -/** - * pm_genpd_default_freeze - Default "device freeze" for PM domians. - * @dev: Device to handle. - */ -static int pm_genpd_default_freeze(struct device *dev) -{ - int (*cb)(struct device *__dev) = dev_gpd_data(dev)->ops.freeze; - - return cb ? cb(dev) : pm_generic_freeze(dev); -} - -/** - * pm_genpd_default_freeze_late - Default "late device freeze" for PM domians. - * @dev: Device to handle. - */ -static int pm_genpd_default_freeze_late(struct device *dev) -{ - int (*cb)(struct device *__dev) = dev_gpd_data(dev)->ops.freeze_late; - - return cb ? cb(dev) : pm_generic_freeze_late(dev); -} - -/** - * pm_genpd_default_thaw_early - Default "early device thaw" for PM domians. - * @dev: Device to handle. - */ -static int pm_genpd_default_thaw_early(struct device *dev) -{ - int (*cb)(struct device *__dev) = dev_gpd_data(dev)->ops.thaw_early; - - return cb ? cb(dev) : pm_generic_thaw_early(dev); -} - -/** - * pm_genpd_default_thaw - Default "device thaw" for PM domians. - * @dev: Device to handle. - */ -static int pm_genpd_default_thaw(struct device *dev) -{ - int (*cb)(struct device *__dev) = dev_gpd_data(dev)->ops.thaw; - - return cb ? cb(dev) : pm_generic_thaw(dev); -} - -#else /* !CONFIG_PM_SLEEP */ - -#define pm_genpd_default_suspend NULL -#define pm_genpd_default_suspend_late NULL -#define pm_genpd_default_resume_early NULL -#define pm_genpd_default_resume NULL -#define pm_genpd_default_freeze NULL -#define pm_genpd_default_freeze_late NULL -#define pm_genpd_default_thaw_early NULL -#define pm_genpd_default_thaw NULL - -#endif /* !CONFIG_PM_SLEEP */ - -/** - * pm_genpd_init - Initialize a generic I/O PM domain object. - * @genpd: PM domain object to initialize. - * @gov: PM domain governor to associate with the domain (may be NULL). - * @is_off: Initial value of the domain's power_is_off field. - */ -void pm_genpd_init(struct generic_pm_domain *genpd, - struct dev_power_governor *gov, bool is_off) -{ - if (IS_ERR_OR_NULL(genpd)) - return; - - INIT_LIST_HEAD(&genpd->master_links); - INIT_LIST_HEAD(&genpd->slave_links); - INIT_LIST_HEAD(&genpd->dev_list); - mutex_init(&genpd->lock); - genpd->gov = gov; - INIT_WORK(&genpd->power_off_work, genpd_power_off_work_fn); - genpd->in_progress = 0; - atomic_set(&genpd->sd_count, 0); - genpd->status = is_off ? GPD_STATE_POWER_OFF : GPD_STATE_ACTIVE; - init_waitqueue_head(&genpd->status_wait_queue); - genpd->poweroff_task = NULL; - genpd->resume_count = 0; - genpd->device_count = 0; - genpd->max_off_time_ns = -1; - genpd->max_off_time_changed = true; - genpd->domain.ops.runtime_suspend = pm_genpd_runtime_suspend; - genpd->domain.ops.runtime_resume = pm_genpd_runtime_resume; - genpd->domain.ops.prepare = pm_genpd_prepare; - genpd->domain.ops.suspend = pm_genpd_suspend; - genpd->domain.ops.suspend_late = pm_genpd_suspend_late; - genpd->domain.ops.suspend_noirq = pm_genpd_suspend_noirq; - genpd->domain.ops.resume_noirq = pm_genpd_resume_noirq; - genpd->domain.ops.resume_early = pm_genpd_resume_early; - genpd->domain.ops.resume = pm_genpd_resume; - genpd->domain.ops.freeze = pm_genpd_freeze; - genpd->domain.ops.freeze_late = pm_genpd_freeze_late; - genpd->domain.ops.freeze_noirq = pm_genpd_freeze_noirq; - genpd->domain.ops.thaw_noirq = pm_genpd_thaw_noirq; - genpd->domain.ops.thaw_early = pm_genpd_thaw_early; - genpd->domain.ops.thaw = pm_genpd_thaw; - genpd->domain.ops.poweroff = pm_genpd_suspend; - genpd->domain.ops.poweroff_late = pm_genpd_suspend_late; - genpd->domain.ops.poweroff_noirq = pm_genpd_suspend_noirq; - genpd->domain.ops.restore_noirq = pm_genpd_restore_noirq; - genpd->domain.ops.restore_early = pm_genpd_resume_early; - genpd->domain.ops.restore = pm_genpd_resume; - genpd->domain.ops.complete = pm_genpd_complete; - genpd->dev_ops.save_state = pm_genpd_default_save_state; - genpd->dev_ops.restore_state = pm_genpd_default_restore_state; - genpd->dev_ops.suspend = pm_genpd_default_suspend; - genpd->dev_ops.suspend_late = pm_genpd_default_suspend_late; - genpd->dev_ops.resume_early = pm_genpd_default_resume_early; - genpd->dev_ops.resume = pm_genpd_default_resume; - genpd->dev_ops.freeze = pm_genpd_default_freeze; - genpd->dev_ops.freeze_late = pm_genpd_default_freeze_late; - genpd->dev_ops.thaw_early = pm_genpd_default_thaw_early; - genpd->dev_ops.thaw = pm_genpd_default_thaw; - mutex_lock(&gpd_list_lock); - list_add(&genpd->gpd_list_node, &gpd_list); - mutex_unlock(&gpd_list_lock); -} diff --git a/drivers/base/power/domain_governor.c b/drivers/base/power/domain_governor.c deleted file mode 100644 index 28dee3053f1f..000000000000 --- a/drivers/base/power/domain_governor.c +++ /dev/null @@ -1,254 +0,0 @@ -/* - * drivers/base/power/domain_governor.c - Governors for device PM domains. - * - * Copyright (C) 2011 Rafael J. Wysocki <rjw@sisk.pl>, Renesas Electronics Corp. - * - * This file is released under the GPLv2. - */ - -#include <linux/init.h> -#include <linux/kernel.h> -#include <linux/pm_domain.h> -#include <linux/pm_qos.h> -#include <linux/hrtimer.h> - -#ifdef CONFIG_PM_RUNTIME - -static int dev_update_qos_constraint(struct device *dev, void *data) -{ - s64 *constraint_ns_p = data; - s32 constraint_ns = -1; - - if (dev->power.subsys_data && dev->power.subsys_data->domain_data) - constraint_ns = dev_gpd_data(dev)->td.effective_constraint_ns; - - if (constraint_ns < 0) { - constraint_ns = dev_pm_qos_read_value(dev); - constraint_ns *= NSEC_PER_USEC; - } - if (constraint_ns == 0) - return 0; - - /* - * constraint_ns cannot be negative here, because the device has been - * suspended. - */ - if (constraint_ns < *constraint_ns_p || *constraint_ns_p == 0) - *constraint_ns_p = constraint_ns; - - return 0; -} - -/** - * default_stop_ok - Default PM domain governor routine for stopping devices. - * @dev: Device to check. - */ -bool default_stop_ok(struct device *dev) -{ - struct gpd_timing_data *td = &dev_gpd_data(dev)->td; - unsigned long flags; - s64 constraint_ns; - - dev_dbg(dev, "%s()\n", __func__); - - spin_lock_irqsave(&dev->power.lock, flags); - - if (!td->constraint_changed) { - bool ret = td->cached_stop_ok; - - spin_unlock_irqrestore(&dev->power.lock, flags); - return ret; - } - td->constraint_changed = false; - td->cached_stop_ok = false; - td->effective_constraint_ns = -1; - constraint_ns = __dev_pm_qos_read_value(dev); - - spin_unlock_irqrestore(&dev->power.lock, flags); - - if (constraint_ns < 0) - return false; - - constraint_ns *= NSEC_PER_USEC; - /* - * We can walk the children without any additional locking, because - * they all have been suspended at this point and their - * effective_constraint_ns fields won't be modified in parallel with us. - */ - if (!dev->power.ignore_children) - device_for_each_child(dev, &constraint_ns, - dev_update_qos_constraint); - - if (constraint_ns > 0) { - constraint_ns -= td->start_latency_ns; - if (constraint_ns == 0) - return false; - } - td->effective_constraint_ns = constraint_ns; - td->cached_stop_ok = constraint_ns > td->stop_latency_ns || - constraint_ns == 0; - /* - * The children have been suspended already, so we don't need to take - * their stop latencies into account here. - */ - return td->cached_stop_ok; -} - -/** - * default_power_down_ok - Default generic PM domain power off governor routine. - * @pd: PM domain to check. - * - * This routine must be executed under the PM domain's lock. - */ -static bool default_power_down_ok(struct dev_pm_domain *pd) -{ - struct generic_pm_domain *genpd = pd_to_genpd(pd); - struct gpd_link *link; - struct pm_domain_data *pdd; - s64 min_off_time_ns; - s64 off_on_time_ns; - - if (genpd->max_off_time_changed) { - struct gpd_link *link; - - /* - * We have to invalidate the cached results for the masters, so - * use the observation that default_power_down_ok() is not - * going to be called for any master until this instance - * returns. - */ - list_for_each_entry(link, &genpd->slave_links, slave_node) - link->master->max_off_time_changed = true; - - genpd->max_off_time_changed = false; - genpd->cached_power_down_ok = false; - genpd->max_off_time_ns = -1; - } else { - return genpd->cached_power_down_ok; - } - - off_on_time_ns = genpd->power_off_latency_ns + - genpd->power_on_latency_ns; - /* - * It doesn't make sense to remove power from the domain if saving - * the state of all devices in it and the power off/power on operations - * take too much time. - * - * All devices in this domain have been stopped already at this point. - */ - list_for_each_entry(pdd, &genpd->dev_list, list_node) { - if (pdd->dev->driver) - off_on_time_ns += - to_gpd_data(pdd)->td.save_state_latency_ns; - } - - min_off_time_ns = -1; - /* - * Check if subdomains can be off for enough time. - * - * All subdomains have been powered off already at this point. - */ - list_for_each_entry(link, &genpd->master_links, master_node) { - struct generic_pm_domain *sd = link->slave; - s64 sd_max_off_ns = sd->max_off_time_ns; - - if (sd_max_off_ns < 0) - continue; - - /* - * Check if the subdomain is allowed to be off long enough for - * the current domain to turn off and on (that's how much time - * it will have to wait worst case). - */ - if (sd_max_off_ns <= off_on_time_ns) - return false; - - if (min_off_time_ns > sd_max_off_ns || min_off_time_ns < 0) - min_off_time_ns = sd_max_off_ns; - } - - /* - * Check if the devices in the domain can be off enough time. - */ - list_for_each_entry(pdd, &genpd->dev_list, list_node) { - struct gpd_timing_data *td; - s64 constraint_ns; - - if (!pdd->dev->driver) - continue; - - /* - * Check if the device is allowed to be off long enough for the - * domain to turn off and on (that's how much time it will - * have to wait worst case). - */ - td = &to_gpd_data(pdd)->td; - constraint_ns = td->effective_constraint_ns; - /* default_stop_ok() need not be called before us. */ - if (constraint_ns < 0) { - constraint_ns = dev_pm_qos_read_value(pdd->dev); - constraint_ns *= NSEC_PER_USEC; - } - if (constraint_ns == 0) - continue; - - /* - * constraint_ns cannot be negative here, because the device has - * been suspended. - */ - constraint_ns -= td->restore_state_latency_ns; - if (constraint_ns <= off_on_time_ns) - return false; - - if (min_off_time_ns > constraint_ns || min_off_time_ns < 0) - min_off_time_ns = constraint_ns; - } - - genpd->cached_power_down_ok = true; - - /* - * If the computed minimum device off time is negative, there are no - * latency constraints, so the domain can spend arbitrary time in the - * "off" state. - */ - if (min_off_time_ns < 0) - return true; - - /* - * The difference between the computed minimum subdomain or device off - * time and the time needed to turn the domain on is the maximum - * theoretical time this domain can spend in the "off" state. - */ - genpd->max_off_time_ns = min_off_time_ns - genpd->power_on_latency_ns; - return true; -} - -static bool always_on_power_down_ok(struct dev_pm_domain *domain) -{ - return false; -} - -#else /* !CONFIG_PM_RUNTIME */ - -bool default_stop_ok(struct device *dev) -{ - return false; -} - -#define default_power_down_ok NULL -#define always_on_power_down_ok NULL - -#endif /* !CONFIG_PM_RUNTIME */ - -struct dev_power_governor simple_qos_governor = { - .stop_ok = default_stop_ok, - .power_down_ok = default_power_down_ok, -}; - -/** - * pm_genpd_gov_always_on - A governor implementing an always-on policy - */ -struct dev_power_governor pm_domain_always_on_gov = { - .power_down_ok = always_on_power_down_ok, - .stop_ok = default_stop_ok, -}; diff --git a/drivers/base/power/generic_ops.c b/drivers/base/power/generic_ops.c index 5ee030a864f9..af99bbcf281c 100644 --- a/drivers/base/power/generic_ops.c +++ b/drivers/base/power/generic_ops.c @@ -1,16 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0 /* * drivers/base/power/generic_ops.c - Generic PM callbacks for subsystems * * Copyright (c) 2010 Rafael J. Wysocki <rjw@sisk.pl>, Novell Inc. - * - * This file is released under the GPLv2. */ - #include <linux/pm.h> #include <linux/pm_runtime.h> #include <linux/export.h> -#ifdef CONFIG_PM_RUNTIME +#define CALL_PM_OP(dev, op) \ +({ \ + struct device *_dev = (dev); \ + const struct dev_pm_ops *pm = _dev->driver ? _dev->driver->pm : NULL; \ + pm && pm->op ? pm->op(_dev) : 0; \ +}) + +#ifdef CONFIG_PM /** * pm_generic_runtime_suspend - Generic runtime suspend callback for subsystems. * @dev: Device to suspend. @@ -21,12 +26,7 @@ */ int pm_generic_runtime_suspend(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - int ret; - - ret = pm && pm->runtime_suspend ? pm->runtime_suspend(dev) : 0; - - return ret; + return CALL_PM_OP(dev, runtime_suspend); } EXPORT_SYMBOL_GPL(pm_generic_runtime_suspend); @@ -40,15 +40,10 @@ EXPORT_SYMBOL_GPL(pm_generic_runtime_suspend); */ int pm_generic_runtime_resume(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - int ret; - - ret = pm && pm->runtime_resume ? pm->runtime_resume(dev) : 0; - - return ret; + return CALL_PM_OP(dev, runtime_resume); } EXPORT_SYMBOL_GPL(pm_generic_runtime_resume); -#endif /* CONFIG_PM_RUNTIME */ +#endif /* CONFIG_PM */ #ifdef CONFIG_PM_SLEEP /** @@ -74,9 +69,7 @@ int pm_generic_prepare(struct device *dev) */ int pm_generic_suspend_noirq(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - - return pm && pm->suspend_noirq ? pm->suspend_noirq(dev) : 0; + return CALL_PM_OP(dev, suspend_noirq); } EXPORT_SYMBOL_GPL(pm_generic_suspend_noirq); @@ -86,9 +79,7 @@ EXPORT_SYMBOL_GPL(pm_generic_suspend_noirq); */ int pm_generic_suspend_late(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - - return pm && pm->suspend_late ? pm->suspend_late(dev) : 0; + return CALL_PM_OP(dev, suspend_late); } EXPORT_SYMBOL_GPL(pm_generic_suspend_late); @@ -98,9 +89,7 @@ EXPORT_SYMBOL_GPL(pm_generic_suspend_late); */ int pm_generic_suspend(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - - return pm && pm->suspend ? pm->suspend(dev) : 0; + return CALL_PM_OP(dev, suspend); } EXPORT_SYMBOL_GPL(pm_generic_suspend); @@ -110,33 +99,17 @@ EXPORT_SYMBOL_GPL(pm_generic_suspend); */ int pm_generic_freeze_noirq(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - - return pm && pm->freeze_noirq ? pm->freeze_noirq(dev) : 0; + return CALL_PM_OP(dev, freeze_noirq); } EXPORT_SYMBOL_GPL(pm_generic_freeze_noirq); /** - * pm_generic_freeze_late - Generic freeze_late callback for subsystems. - * @dev: Device to freeze. - */ -int pm_generic_freeze_late(struct device *dev) -{ - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - - return pm && pm->freeze_late ? pm->freeze_late(dev) : 0; -} -EXPORT_SYMBOL_GPL(pm_generic_freeze_late); - -/** * pm_generic_freeze - Generic freeze callback for subsystems. * @dev: Device to freeze. */ int pm_generic_freeze(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - - return pm && pm->freeze ? pm->freeze(dev) : 0; + return CALL_PM_OP(dev, freeze); } EXPORT_SYMBOL_GPL(pm_generic_freeze); @@ -146,9 +119,7 @@ EXPORT_SYMBOL_GPL(pm_generic_freeze); */ int pm_generic_poweroff_noirq(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - - return pm && pm->poweroff_noirq ? pm->poweroff_noirq(dev) : 0; + return CALL_PM_OP(dev, poweroff_noirq); } EXPORT_SYMBOL_GPL(pm_generic_poweroff_noirq); @@ -158,9 +129,7 @@ EXPORT_SYMBOL_GPL(pm_generic_poweroff_noirq); */ int pm_generic_poweroff_late(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - - return pm && pm->poweroff_late ? pm->poweroff_late(dev) : 0; + return CALL_PM_OP(dev, poweroff_late); } EXPORT_SYMBOL_GPL(pm_generic_poweroff_late); @@ -170,9 +139,7 @@ EXPORT_SYMBOL_GPL(pm_generic_poweroff_late); */ int pm_generic_poweroff(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - - return pm && pm->poweroff ? pm->poweroff(dev) : 0; + return CALL_PM_OP(dev, poweroff); } EXPORT_SYMBOL_GPL(pm_generic_poweroff); @@ -182,33 +149,17 @@ EXPORT_SYMBOL_GPL(pm_generic_poweroff); */ int pm_generic_thaw_noirq(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - - return pm && pm->thaw_noirq ? pm->thaw_noirq(dev) : 0; + return CALL_PM_OP(dev, thaw_noirq); } EXPORT_SYMBOL_GPL(pm_generic_thaw_noirq); /** - * pm_generic_thaw_early - Generic thaw_early callback for subsystems. - * @dev: Device to thaw. - */ -int pm_generic_thaw_early(struct device *dev) -{ - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - - return pm && pm->thaw_early ? pm->thaw_early(dev) : 0; -} -EXPORT_SYMBOL_GPL(pm_generic_thaw_early); - -/** * pm_generic_thaw - Generic thaw callback for subsystems. * @dev: Device to thaw. */ int pm_generic_thaw(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - - return pm && pm->thaw ? pm->thaw(dev) : 0; + return CALL_PM_OP(dev, thaw); } EXPORT_SYMBOL_GPL(pm_generic_thaw); @@ -218,9 +169,7 @@ EXPORT_SYMBOL_GPL(pm_generic_thaw); */ int pm_generic_resume_noirq(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - - return pm && pm->resume_noirq ? pm->resume_noirq(dev) : 0; + return CALL_PM_OP(dev, resume_noirq); } EXPORT_SYMBOL_GPL(pm_generic_resume_noirq); @@ -230,9 +179,7 @@ EXPORT_SYMBOL_GPL(pm_generic_resume_noirq); */ int pm_generic_resume_early(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - - return pm && pm->resume_early ? pm->resume_early(dev) : 0; + return CALL_PM_OP(dev, resume_early); } EXPORT_SYMBOL_GPL(pm_generic_resume_early); @@ -242,9 +189,7 @@ EXPORT_SYMBOL_GPL(pm_generic_resume_early); */ int pm_generic_resume(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - - return pm && pm->resume ? pm->resume(dev) : 0; + return CALL_PM_OP(dev, resume); } EXPORT_SYMBOL_GPL(pm_generic_resume); @@ -254,9 +199,7 @@ EXPORT_SYMBOL_GPL(pm_generic_resume); */ int pm_generic_restore_noirq(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - - return pm && pm->restore_noirq ? pm->restore_noirq(dev) : 0; + return CALL_PM_OP(dev, restore_noirq); } EXPORT_SYMBOL_GPL(pm_generic_restore_noirq); @@ -266,9 +209,7 @@ EXPORT_SYMBOL_GPL(pm_generic_restore_noirq); */ int pm_generic_restore_early(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - - return pm && pm->restore_early ? pm->restore_early(dev) : 0; + return CALL_PM_OP(dev, restore_early); } EXPORT_SYMBOL_GPL(pm_generic_restore_early); @@ -278,14 +219,12 @@ EXPORT_SYMBOL_GPL(pm_generic_restore_early); */ int pm_generic_restore(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - - return pm && pm->restore ? pm->restore(dev) : 0; + return CALL_PM_OP(dev, restore); } EXPORT_SYMBOL_GPL(pm_generic_restore); /** - * pm_generic_complete - Generic routine competing a device power transition. + * pm_generic_complete - Generic routine completing a device power transition. * @dev: Device to handle. * * Complete a device power transition during a system-wide power transition. @@ -296,11 +235,5 @@ void pm_generic_complete(struct device *dev) if (drv && drv->pm && drv->pm->complete) drv->pm->complete(dev); - - /* - * Let runtime PM try to suspend devices that haven't been in use before - * going into the system-wide sleep state we're resuming from. - */ - pm_request_idle(dev); } #endif /* CONFIG_PM_SLEEP */ diff --git a/drivers/base/power/main.c b/drivers/base/power/main.c index 5a9b6569dd74..97a8b4fcf471 100644 --- a/drivers/base/power/main.c +++ b/drivers/base/power/main.c @@ -1,12 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0 /* * drivers/base/power/main.c - Where the driver meets power management. * * Copyright (c) 2003 Patrick Mochel * Copyright (c) 2003 Open Source Development Lab * - * This file is released under the GPLv2 - * - * * The driver model core calls device_pm_add() when a device is registered. * This will initialize the embedded device_pm_info object in the device * and add it to the list of power-controlled devices. sysfs entries for @@ -17,18 +15,27 @@ * subsystem list maintains. */ +#define pr_fmt(fmt) "PM: " fmt +#define dev_fmt pr_fmt + #include <linux/device.h> -#include <linux/kallsyms.h> #include <linux/export.h> #include <linux/mutex.h> #include <linux/pm.h> #include <linux/pm_runtime.h> -#include <linux/resume-trace.h> +#include <linux/pm-trace.h> +#include <linux/pm_wakeirq.h> #include <linux/interrupt.h> #include <linux/sched.h> +#include <linux/sched/debug.h> #include <linux/async.h> #include <linux/suspend.h> -#include <linux/cpuidle.h> +#include <trace/events/power.h> +#include <linux/cpufreq.h> +#include <linux/devfreq.h> +#include <linux/timer.h> +#include <linux/nmi.h> + #include "../base.h" #include "power.h" @@ -50,13 +57,53 @@ static LIST_HEAD(dpm_suspended_list); static LIST_HEAD(dpm_late_early_list); static LIST_HEAD(dpm_noirq_list); -struct suspend_stats suspend_stats; static DEFINE_MUTEX(dpm_list_mtx); static pm_message_t pm_transition; +static DEFINE_MUTEX(async_wip_mtx); static int async_error; /** + * pm_hibernate_is_recovering - if recovering from hibernate due to error. + * + * Used to query if dev_pm_ops.thaw() is called for normal hibernation case or + * recovering from some error. + * + * Return: true for error case, false for normal case. + */ +bool pm_hibernate_is_recovering(void) +{ + return pm_transition.event == PM_EVENT_RECOVER; +} +EXPORT_SYMBOL_GPL(pm_hibernate_is_recovering); + +static const char *pm_verb(int event) +{ + switch (event) { + case PM_EVENT_SUSPEND: + return "suspend"; + case PM_EVENT_RESUME: + return "resume"; + case PM_EVENT_FREEZE: + return "freeze"; + case PM_EVENT_QUIESCE: + return "quiesce"; + case PM_EVENT_HIBERNATE: + return "hibernate"; + case PM_EVENT_THAW: + return "thaw"; + case PM_EVENT_RESTORE: + return "restore"; + case PM_EVENT_RECOVER: + return "recover"; + case PM_EVENT_POWEROFF: + return "poweroff"; + default: + return "(unknown PM event)"; + } +} + +/** * device_pm_sleep_init - Initialize system suspend-related device fields. * @dev: Device object being initialized. */ @@ -64,6 +111,8 @@ void device_pm_sleep_init(struct device *dev) { dev->power.is_prepared = false; dev->power.is_suspended = false; + dev->power.is_noirq_suspended = false; + dev->power.is_late_suspended = false; init_completion(&dev->power.completion); complete_all(&dev->power.completion); dev->power.wakeup = NULL; @@ -92,13 +141,19 @@ void device_pm_unlock(void) */ void device_pm_add(struct device *dev) { - pr_debug("PM: Adding info for %s:%s\n", + /* Skip PM setup/initialization. */ + if (device_pm_not_required(dev)) + return; + + pr_debug("Adding info for %s:%s\n", dev->bus ? dev->bus->name : "No Bus", dev_name(dev)); + device_pm_check_callbacks(dev); mutex_lock(&dpm_list_mtx); if (dev->parent && dev->parent->power.is_prepared) dev_warn(dev, "parent %s should not be sleeping\n", dev_name(dev->parent)); list_add_tail(&dev->power.entry, &dpm_list); + dev->power.in_dpm_list = true; mutex_unlock(&dpm_list_mtx); } @@ -108,14 +163,19 @@ void device_pm_add(struct device *dev) */ void device_pm_remove(struct device *dev) { - pr_debug("PM: Removing info for %s:%s\n", + if (device_pm_not_required(dev)) + return; + + pr_debug("Removing info for %s:%s\n", dev->bus ? dev->bus->name : "No Bus", dev_name(dev)); complete_all(&dev->power.completion); mutex_lock(&dpm_list_mtx); list_del_init(&dev->power.entry); + dev->power.in_dpm_list = false; mutex_unlock(&dpm_list_mtx); device_wakeup_disable(dev); pm_runtime_remove(dev); + device_pm_check_callbacks(dev); } /** @@ -125,7 +185,7 @@ void device_pm_remove(struct device *dev) */ void device_pm_move_before(struct device *deva, struct device *devb) { - pr_debug("PM: Moving %s:%s before %s:%s\n", + pr_debug("Moving %s:%s before %s:%s\n", deva->bus ? deva->bus->name : "No Bus", dev_name(deva), devb->bus ? devb->bus->name : "No Bus", dev_name(devb)); /* Delete deva from dpm_list and reinsert before devb. */ @@ -139,7 +199,7 @@ void device_pm_move_before(struct device *deva, struct device *devb) */ void device_pm_move_after(struct device *deva, struct device *devb) { - pr_debug("PM: Moving %s:%s after %s:%s\n", + pr_debug("Moving %s:%s after %s:%s\n", deva->bus ? deva->bus->name : "No Bus", dev_name(deva), devb->bus ? devb->bus->name : "No Bus", dev_name(devb)); /* Delete deva from dpm_list and reinsert after devb. */ @@ -152,36 +212,33 @@ void device_pm_move_after(struct device *deva, struct device *devb) */ void device_pm_move_last(struct device *dev) { - pr_debug("PM: Moving %s:%s to end of list\n", + pr_debug("Moving %s:%s to end of list\n", dev->bus ? dev->bus->name : "No Bus", dev_name(dev)); list_move_tail(&dev->power.entry, &dpm_list); } -static ktime_t initcall_debug_start(struct device *dev) +static ktime_t initcall_debug_start(struct device *dev, void *cb) { - ktime_t calltime = ktime_set(0, 0); - - if (pm_print_times_enabled) { - pr_info("calling %s+ @ %i, parent: %s\n", - dev_name(dev), task_pid_nr(current), - dev->parent ? dev_name(dev->parent) : "none"); - calltime = ktime_get(); - } + if (!pm_print_times_enabled) + return 0; - return calltime; + dev_info(dev, "calling %ps @ %i, parent: %s\n", cb, + task_pid_nr(current), + dev->parent ? dev_name(dev->parent) : "none"); + return ktime_get(); } static void initcall_debug_report(struct device *dev, ktime_t calltime, - int error) + void *cb, int error) { - ktime_t delta, rettime; + ktime_t rettime; - if (pm_print_times_enabled) { - rettime = ktime_get(); - delta = ktime_sub(rettime, calltime); - pr_info("call %s+ returned %d after %Ld usecs\n", dev_name(dev), - error, (unsigned long long)ktime_to_ns(delta) >> 10); - } + if (!pm_print_times_enabled) + return; + + rettime = ktime_get(); + dev_info(dev, "%ps returned %d after %Ld usecs\n", cb, error, + (unsigned long long)ktime_us_delta(rettime, calltime)); } /** @@ -206,7 +263,93 @@ static int dpm_wait_fn(struct device *dev, void *async_ptr) static void dpm_wait_for_children(struct device *dev, bool async) { - device_for_each_child(dev, &async, dpm_wait_fn); + device_for_each_child(dev, &async, dpm_wait_fn); +} + +static void dpm_wait_for_suppliers(struct device *dev, bool async) +{ + struct device_link *link; + int idx; + + idx = device_links_read_lock(); + + /* + * If the supplier goes away right after we've checked the link to it, + * we'll wait for its completion to change the state, but that's fine, + * because the only things that will block as a result are the SRCU + * callbacks freeing the link objects for the links in the list we're + * walking. + */ + dev_for_each_link_to_supplier(link, dev) + if (READ_ONCE(link->status) != DL_STATE_DORMANT && + !device_link_flag_is_sync_state_only(link->flags)) + dpm_wait(link->supplier, async); + + device_links_read_unlock(idx); +} + +static bool dpm_wait_for_superior(struct device *dev, bool async) +{ + struct device *parent; + + /* + * If the device is resumed asynchronously and the parent's callback + * deletes both the device and the parent itself, the parent object may + * be freed while this function is running, so avoid that by reference + * counting the parent once more unless the device has been deleted + * already (in which case return right away). + */ + mutex_lock(&dpm_list_mtx); + + if (!device_pm_initialized(dev)) { + mutex_unlock(&dpm_list_mtx); + return false; + } + + parent = get_device(dev->parent); + + mutex_unlock(&dpm_list_mtx); + + dpm_wait(parent, async); + put_device(parent); + + dpm_wait_for_suppliers(dev, async); + + /* + * If the parent's callback has deleted the device, attempting to resume + * it would be invalid, so avoid doing that then. + */ + return device_pm_initialized(dev); +} + +static void dpm_wait_for_consumers(struct device *dev, bool async) +{ + struct device_link *link; + int idx; + + idx = device_links_read_lock(); + + /* + * The status of a device link can only be changed from "dormant" by a + * probe, but that cannot happen during system suspend/resume. In + * theory it can change to "dormant" at that time, but then it is + * reasonable to wait for the target device anyway (eg. if it goes + * away, it's better to wait for it to go away completely and then + * continue instead of trying to continue in parallel with its + * unregistration). + */ + dev_for_each_link_to_consumer(link, dev) + if (READ_ONCE(link->status) != DL_STATE_DORMANT && + !device_link_flag_is_sync_state_only(link->flags)) + dpm_wait(link->consumer, async); + + device_links_read_unlock(idx); +} + +static void dpm_wait_for_subordinate(struct device *dev, bool async) +{ + dpm_wait_for_children(dev, async); + dpm_wait_for_consumers(dev, async); } /** @@ -227,12 +370,12 @@ static pm_callback_t pm_op(const struct dev_pm_ops *ops, pm_message_t state) case PM_EVENT_FREEZE: case PM_EVENT_QUIESCE: return ops->freeze; + case PM_EVENT_POWEROFF: case PM_EVENT_HIBERNATE: return ops->poweroff; case PM_EVENT_THAW: case PM_EVENT_RECOVER: return ops->thaw; - break; case PM_EVENT_RESTORE: return ops->restore; #endif /* CONFIG_HIBERNATE_CALLBACKS */ @@ -262,6 +405,7 @@ static pm_callback_t pm_late_early_op(const struct dev_pm_ops *ops, case PM_EVENT_FREEZE: case PM_EVENT_QUIESCE: return ops->freeze_late; + case PM_EVENT_POWEROFF: case PM_EVENT_HIBERNATE: return ops->poweroff_late; case PM_EVENT_THAW: @@ -296,6 +440,7 @@ static pm_callback_t pm_noirq_op(const struct dev_pm_ops *ops, pm_message_t stat case PM_EVENT_FREEZE: case PM_EVENT_QUIESCE: return ops->freeze_noirq; + case PM_EVENT_POWEROFF: case PM_EVENT_HIBERNATE: return ops->poweroff_noirq; case PM_EVENT_THAW: @@ -309,45 +454,22 @@ static pm_callback_t pm_noirq_op(const struct dev_pm_ops *ops, pm_message_t stat return NULL; } -static char *pm_verb(int event) +static void pm_dev_dbg(struct device *dev, pm_message_t state, const char *info) { - switch (event) { - case PM_EVENT_SUSPEND: - return "suspend"; - case PM_EVENT_RESUME: - return "resume"; - case PM_EVENT_FREEZE: - return "freeze"; - case PM_EVENT_QUIESCE: - return "quiesce"; - case PM_EVENT_HIBERNATE: - return "hibernate"; - case PM_EVENT_THAW: - return "thaw"; - case PM_EVENT_RESTORE: - return "restore"; - case PM_EVENT_RECOVER: - return "recover"; - default: - return "(unknown PM event)"; - } -} - -static void pm_dev_dbg(struct device *dev, pm_message_t state, char *info) -{ - dev_dbg(dev, "%s%s%s\n", info, pm_verb(state.event), + dev_dbg(dev, "%s%s%s driver flags: %x\n", info, pm_verb(state.event), ((state.event & PM_EVENT_SLEEP) && device_may_wakeup(dev)) ? - ", may wakeup" : ""); + ", may wakeup" : "", dev->power.driver_flags); } -static void pm_dev_err(struct device *dev, pm_message_t state, char *info, +static void pm_dev_err(struct device *dev, pm_message_t state, const char *info, int error) { - printk(KERN_ERR "PM: Device %s failed to %s%s: error %d\n", - dev_name(dev), pm_verb(state.event), info, error); + dev_err(dev, "failed to %s%s: error %d\n", pm_verb(state.event), info, + error); } -static void dpm_show_time(ktime_t starttime, pm_message_t state, char *info) +static void dpm_show_time(ktime_t starttime, pm_message_t state, int error, + const char *info) { ktime_t calltime; u64 usecs64; @@ -359,13 +481,15 @@ static void dpm_show_time(ktime_t starttime, pm_message_t state, char *info) usecs = usecs64; if (usecs == 0) usecs = 1; - pr_info("PM: %s%s%s of devices complete after %ld.%03ld msecs\n", - info ?: "", info ? " " : "", pm_verb(state.event), - usecs / USEC_PER_MSEC, usecs % USEC_PER_MSEC); + + pm_pr_dbg("%s%s%s of devices %s after %ld.%03ld msecs\n", + info ?: "", info ? " " : "", pm_verb(state.event), + error ? "aborted" : "complete", + usecs / USEC_PER_MSEC, usecs % USEC_PER_MSEC); } static int dpm_run_callback(pm_callback_t cb, struct device *dev, - pm_message_t state, char *info) + pm_message_t state, const char *info) { ktime_t calltime; int error; @@ -373,38 +497,279 @@ static int dpm_run_callback(pm_callback_t cb, struct device *dev, if (!cb) return 0; - calltime = initcall_debug_start(dev); + calltime = initcall_debug_start(dev, cb); pm_dev_dbg(dev, state, info); + trace_device_pm_callback_start(dev, info, state.event); error = cb(dev); - suspend_report_result(cb, error); + trace_device_pm_callback_end(dev, error); + suspend_report_result(dev, cb, error); - initcall_debug_report(dev, calltime, error); + initcall_debug_report(dev, calltime, cb, error); return error; } +#ifdef CONFIG_DPM_WATCHDOG +struct dpm_watchdog { + struct device *dev; + struct task_struct *tsk; + struct timer_list timer; + bool fatal; +}; + +#define DECLARE_DPM_WATCHDOG_ON_STACK(wd) \ + struct dpm_watchdog wd + +static bool __read_mostly dpm_watchdog_all_cpu_backtrace; +module_param(dpm_watchdog_all_cpu_backtrace, bool, 0644); +MODULE_PARM_DESC(dpm_watchdog_all_cpu_backtrace, + "Backtrace all CPUs on DPM watchdog timeout"); + +/** + * dpm_watchdog_handler - Driver suspend / resume watchdog handler. + * @t: The timer that PM watchdog depends on. + * + * Called when a driver has timed out suspending or resuming. + * There's not much we can do here to recover so panic() to + * capture a crash-dump in pstore. + */ +static void dpm_watchdog_handler(struct timer_list *t) +{ + struct dpm_watchdog *wd = timer_container_of(wd, t, timer); + struct timer_list *timer = &wd->timer; + unsigned int time_left; + + if (wd->fatal) { + unsigned int this_cpu = smp_processor_id(); + + dev_emerg(wd->dev, "**** DPM device timeout ****\n"); + show_stack(wd->tsk, NULL, KERN_EMERG); + if (dpm_watchdog_all_cpu_backtrace) + trigger_allbutcpu_cpu_backtrace(this_cpu); + panic("%s %s: unrecoverable failure\n", + dev_driver_string(wd->dev), dev_name(wd->dev)); + } + + time_left = CONFIG_DPM_WATCHDOG_TIMEOUT - CONFIG_DPM_WATCHDOG_WARNING_TIMEOUT; + dev_warn(wd->dev, "**** DPM device timeout after %u seconds; %u seconds until panic ****\n", + CONFIG_DPM_WATCHDOG_WARNING_TIMEOUT, time_left); + show_stack(wd->tsk, NULL, KERN_WARNING); + + wd->fatal = true; + mod_timer(timer, jiffies + HZ * time_left); +} + +/** + * dpm_watchdog_set - Enable pm watchdog for given device. + * @wd: Watchdog. Must be allocated on the stack. + * @dev: Device to handle. + */ +static void dpm_watchdog_set(struct dpm_watchdog *wd, struct device *dev) +{ + struct timer_list *timer = &wd->timer; + + wd->dev = dev; + wd->tsk = current; + wd->fatal = CONFIG_DPM_WATCHDOG_TIMEOUT == CONFIG_DPM_WATCHDOG_WARNING_TIMEOUT; + + timer_setup_on_stack(timer, dpm_watchdog_handler, 0); + /* use same timeout value for both suspend and resume */ + timer->expires = jiffies + HZ * CONFIG_DPM_WATCHDOG_WARNING_TIMEOUT; + add_timer(timer); +} + +/** + * dpm_watchdog_clear - Disable suspend/resume watchdog. + * @wd: Watchdog to disable. + */ +static void dpm_watchdog_clear(struct dpm_watchdog *wd) +{ + struct timer_list *timer = &wd->timer; + + timer_delete_sync(timer); + timer_destroy_on_stack(timer); +} +#else +#define DECLARE_DPM_WATCHDOG_ON_STACK(wd) +#define dpm_watchdog_set(x, y) +#define dpm_watchdog_clear(x) +#endif + /*------------------------- Resume routines -------------------------*/ /** - * device_resume_noirq - Execute an "early resume" callback for given device. + * dev_pm_skip_resume - System-wide device resume optimization check. + * @dev: Target device. + * + * Return: + * - %false if the transition under way is RESTORE. + * - Return value of dev_pm_skip_suspend() if the transition under way is THAW. + * - The logical negation of %power.must_resume otherwise (that is, when the + * transition under way is RESUME). + */ +bool dev_pm_skip_resume(struct device *dev) +{ + if (pm_transition.event == PM_EVENT_RESTORE) + return false; + + if (pm_transition.event == PM_EVENT_THAW) + return dev_pm_skip_suspend(dev); + + return !dev->power.must_resume; +} + +static bool is_async(struct device *dev) +{ + return dev->power.async_suspend && pm_async_enabled + && !pm_trace_is_enabled(); +} + +static bool __dpm_async(struct device *dev, async_func_t func) +{ + if (dev->power.work_in_progress) + return true; + + if (!is_async(dev)) + return false; + + dev->power.work_in_progress = true; + + get_device(dev); + + if (async_schedule_dev_nocall(func, dev)) + return true; + + put_device(dev); + + return false; +} + +static bool dpm_async_fn(struct device *dev, async_func_t func) +{ + guard(mutex)(&async_wip_mtx); + + return __dpm_async(dev, func); +} + +static int dpm_async_with_cleanup(struct device *dev, void *fn) +{ + guard(mutex)(&async_wip_mtx); + + if (!__dpm_async(dev, fn)) + dev->power.work_in_progress = false; + + return 0; +} + +static void dpm_async_resume_children(struct device *dev, async_func_t func) +{ + /* + * Prevent racing with dpm_clear_async_state() during initial list + * walks in dpm_noirq_resume_devices(), dpm_resume_early(), and + * dpm_resume(). + */ + guard(mutex)(&dpm_list_mtx); + + /* + * Start processing "async" children of the device unless it's been + * started already for them. + */ + device_for_each_child(dev, func, dpm_async_with_cleanup); +} + +static void dpm_async_resume_subordinate(struct device *dev, async_func_t func) +{ + struct device_link *link; + int idx; + + dpm_async_resume_children(dev, func); + + idx = device_links_read_lock(); + + /* Start processing the device's "async" consumers. */ + dev_for_each_link_to_consumer(link, dev) + if (READ_ONCE(link->status) != DL_STATE_DORMANT) + dpm_async_with_cleanup(link->consumer, func); + + device_links_read_unlock(idx); +} + +static void dpm_clear_async_state(struct device *dev) +{ + reinit_completion(&dev->power.completion); + dev->power.work_in_progress = false; +} + +static bool dpm_root_device(struct device *dev) +{ + lockdep_assert_held(&dpm_list_mtx); + + /* + * Since this function is required to run under dpm_list_mtx, the + * list_empty() below will only return true if the device's list of + * consumers is actually empty before calling it. + */ + return !dev->parent && list_empty(&dev->links.suppliers); +} + +static void async_resume_noirq(void *data, async_cookie_t cookie); + +/** + * device_resume_noirq - Execute a "noirq resume" callback for given device. * @dev: Device to handle. * @state: PM transition of the system being carried out. + * @async: If true, the device is being resumed asynchronously. * * The driver of @dev will not receive interrupts while this function is being * executed. */ -static int device_resume_noirq(struct device *dev, pm_message_t state) +static void device_resume_noirq(struct device *dev, pm_message_t state, bool async) { pm_callback_t callback = NULL; - char *info = NULL; + const char *info = NULL; + bool skip_resume; int error = 0; TRACE_DEVICE(dev); TRACE_RESUME(0); - if (dev->power.syscore) + if (dev->power.syscore || dev->power.direct_complete) + goto Out; + + if (!dev->power.is_noirq_suspended) { + /* + * This means that system suspend has been aborted in the noirq + * phase before invoking the noirq suspend callback for the + * device, so if device_suspend_late() has left it in suspend, + * device_resume_early() should leave it in suspend either in + * case the early resume of it depends on the noirq resume that + * has not run. + */ + if (dev_pm_skip_suspend(dev)) + dev->power.must_resume = false; + goto Out; + } + + if (!dpm_wait_for_superior(dev, async)) + goto Out; + + skip_resume = dev_pm_skip_resume(dev); + /* + * If the driver callback is skipped below or by the middle layer + * callback and device_resume_early() also skips the driver callback for + * this device later, it needs to appear as "suspended" to PM-runtime, + * so change its status accordingly. + * + * Otherwise, the device is going to be resumed, so set its PM-runtime + * status to "active" unless its power.smart_suspend flag is clear, in + * which case it is not necessary to update its PM-runtime status. + */ + if (skip_resume) + pm_runtime_set_suspended(dev); + else if (dev_pm_smart_suspend(dev)) + pm_runtime_set_active(dev); if (dev->pm_domain) { info = "noirq power domain "; @@ -419,73 +784,135 @@ static int device_resume_noirq(struct device *dev, pm_message_t state) info = "noirq bus "; callback = pm_noirq_op(dev->bus->pm, state); } + if (callback) + goto Run; - if (!callback && dev->driver && dev->driver->pm) { + if (skip_resume) + goto Skip; + + if (dev->driver && dev->driver->pm) { info = "noirq driver "; callback = pm_noirq_op(dev->driver->pm, state); } +Run: error = dpm_run_callback(callback, dev, state, info); - Out: +Skip: + dev->power.is_noirq_suspended = false; + +Out: + complete_all(&dev->power.completion); TRACE_RESUME(error); - return error; + + if (error) { + WRITE_ONCE(async_error, error); + dpm_save_failed_dev(dev_name(dev)); + pm_dev_err(dev, state, async ? " async noirq" : " noirq", error); + } + + dpm_async_resume_subordinate(dev, async_resume_noirq); } -/** - * dpm_resume_noirq - Execute "noirq resume" callbacks for all devices. - * @state: PM transition of the system being carried out. - * - * Call the "noirq" resume handlers for all devices in dpm_noirq_list and - * enable device drivers to receive interrupts. - */ -static void dpm_resume_noirq(pm_message_t state) +static void async_resume_noirq(void *data, async_cookie_t cookie) { + struct device *dev = data; + + device_resume_noirq(dev, pm_transition, true); + put_device(dev); +} + +static void dpm_noirq_resume_devices(pm_message_t state) +{ + struct device *dev; ktime_t starttime = ktime_get(); + trace_suspend_resume(TPS("dpm_resume_noirq"), state.event, true); + + async_error = 0; + pm_transition = state; + mutex_lock(&dpm_list_mtx); - while (!list_empty(&dpm_noirq_list)) { - struct device *dev = to_device(dpm_noirq_list.next); - int error; - get_device(dev); + /* + * Start processing "async" root devices upfront so they don't wait for + * the "sync" devices they don't depend on. + */ + list_for_each_entry(dev, &dpm_noirq_list, power.entry) { + dpm_clear_async_state(dev); + if (dpm_root_device(dev)) + dpm_async_with_cleanup(dev, async_resume_noirq); + } + + while (!list_empty(&dpm_noirq_list)) { + dev = to_device(dpm_noirq_list.next); list_move_tail(&dev->power.entry, &dpm_late_early_list); - mutex_unlock(&dpm_list_mtx); - error = device_resume_noirq(dev, state); - if (error) { - suspend_stats.failed_resume_noirq++; - dpm_save_failed_step(SUSPEND_RESUME_NOIRQ); - dpm_save_failed_dev(dev_name(dev)); - pm_dev_err(dev, state, " noirq", error); - } + if (!dpm_async_fn(dev, async_resume_noirq)) { + get_device(dev); - mutex_lock(&dpm_list_mtx); - put_device(dev); + mutex_unlock(&dpm_list_mtx); + + device_resume_noirq(dev, state, false); + + put_device(dev); + + mutex_lock(&dpm_list_mtx); + } } mutex_unlock(&dpm_list_mtx); - dpm_show_time(starttime, state, "noirq"); + async_synchronize_full(); + dpm_show_time(starttime, state, 0, "noirq"); + if (READ_ONCE(async_error)) + dpm_save_failed_step(SUSPEND_RESUME_NOIRQ); + + trace_suspend_resume(TPS("dpm_resume_noirq"), state.event, false); +} + +/** + * dpm_resume_noirq - Execute "noirq resume" callbacks for all devices. + * @state: PM transition of the system being carried out. + * + * Invoke the "noirq" resume callbacks for all devices in dpm_noirq_list and + * allow device drivers' interrupt handlers to be called. + */ +void dpm_resume_noirq(pm_message_t state) +{ + dpm_noirq_resume_devices(state); + resume_device_irqs(); - cpuidle_resume(); + device_wakeup_disarm_wake_irqs(); } +static void async_resume_early(void *data, async_cookie_t cookie); + /** * device_resume_early - Execute an "early resume" callback for given device. * @dev: Device to handle. * @state: PM transition of the system being carried out. + * @async: If true, the device is being resumed asynchronously. * * Runtime PM is disabled for @dev while this function is being executed. */ -static int device_resume_early(struct device *dev, pm_message_t state) +static void device_resume_early(struct device *dev, pm_message_t state, bool async) { pm_callback_t callback = NULL; - char *info = NULL; + const char *info = NULL; int error = 0; TRACE_DEVICE(dev); TRACE_RESUME(0); + if (dev->power.direct_complete) + goto Out; + + if (!dev->power.is_late_suspended) + goto Out; + if (dev->power.syscore) + goto Skip; + + if (!dpm_wait_for_superior(dev, async)) goto Out; if (dev->pm_domain) { @@ -501,51 +928,95 @@ static int device_resume_early(struct device *dev, pm_message_t state) info = "early bus "; callback = pm_late_early_op(dev->bus->pm, state); } + if (callback) + goto Run; - if (!callback && dev->driver && dev->driver->pm) { + if (dev_pm_skip_resume(dev)) + goto Skip; + + if (dev->driver && dev->driver->pm) { info = "early driver "; callback = pm_late_early_op(dev->driver->pm, state); } +Run: error = dpm_run_callback(callback, dev, state, info); - Out: +Skip: + dev->power.is_late_suspended = false; + pm_runtime_enable(dev); + +Out: TRACE_RESUME(error); - pm_runtime_enable(dev); - return error; + complete_all(&dev->power.completion); + + if (error) { + WRITE_ONCE(async_error, error); + dpm_save_failed_dev(dev_name(dev)); + pm_dev_err(dev, state, async ? " async early" : " early", error); + } + + dpm_async_resume_subordinate(dev, async_resume_early); +} + +static void async_resume_early(void *data, async_cookie_t cookie) +{ + struct device *dev = data; + + device_resume_early(dev, pm_transition, true); + put_device(dev); } /** * dpm_resume_early - Execute "early resume" callbacks for all devices. * @state: PM transition of the system being carried out. */ -static void dpm_resume_early(pm_message_t state) +void dpm_resume_early(pm_message_t state) { + struct device *dev; ktime_t starttime = ktime_get(); + trace_suspend_resume(TPS("dpm_resume_early"), state.event, true); + + async_error = 0; + pm_transition = state; + mutex_lock(&dpm_list_mtx); - while (!list_empty(&dpm_late_early_list)) { - struct device *dev = to_device(dpm_late_early_list.next); - int error; - get_device(dev); + /* + * Start processing "async" root devices upfront so they don't wait for + * the "sync" devices they don't depend on. + */ + list_for_each_entry(dev, &dpm_late_early_list, power.entry) { + dpm_clear_async_state(dev); + if (dpm_root_device(dev)) + dpm_async_with_cleanup(dev, async_resume_early); + } + + while (!list_empty(&dpm_late_early_list)) { + dev = to_device(dpm_late_early_list.next); list_move_tail(&dev->power.entry, &dpm_suspended_list); - mutex_unlock(&dpm_list_mtx); - error = device_resume_early(dev, state); - if (error) { - suspend_stats.failed_resume_early++; - dpm_save_failed_step(SUSPEND_RESUME_EARLY); - dpm_save_failed_dev(dev_name(dev)); - pm_dev_err(dev, state, " early", error); - } + if (!dpm_async_fn(dev, async_resume_early)) { + get_device(dev); - mutex_lock(&dpm_list_mtx); - put_device(dev); + mutex_unlock(&dpm_list_mtx); + + device_resume_early(dev, state, false); + + put_device(dev); + + mutex_lock(&dpm_list_mtx); + } } mutex_unlock(&dpm_list_mtx); - dpm_show_time(starttime, state, "early"); + async_synchronize_full(); + dpm_show_time(starttime, state, 0, "early"); + if (READ_ONCE(async_error)) + dpm_save_failed_step(SUSPEND_RESUME_EARLY); + + trace_suspend_resume(TPS("dpm_resume_early"), state.event, false); } /** @@ -559,17 +1030,20 @@ void dpm_resume_start(pm_message_t state) } EXPORT_SYMBOL_GPL(dpm_resume_start); +static void async_resume(void *data, async_cookie_t cookie); + /** * device_resume - Execute "resume" callbacks for given device. * @dev: Device to handle. * @state: PM transition of the system being carried out. * @async: If true, the device is being resumed asynchronously. */ -static int device_resume(struct device *dev, pm_message_t state, bool async) +static void device_resume(struct device *dev, pm_message_t state, bool async) { pm_callback_t callback = NULL; - char *info = NULL; + const char *info = NULL; int error = 0; + DECLARE_DPM_WATCHDOG_ON_STACK(wd); TRACE_DEVICE(dev); TRACE_RESUME(0); @@ -577,7 +1051,28 @@ static int device_resume(struct device *dev, pm_message_t state, bool async) if (dev->power.syscore) goto Complete; - dpm_wait(dev->parent, async); + if (!dev->power.is_suspended) + goto Complete; + + dev->power.is_suspended = false; + + if (dev->power.direct_complete) { + /* + * Allow new children to be added under the device after this + * point if it has no PM callbacks. + */ + if (dev->power.no_pm_callbacks) + dev->power.is_prepared = false; + + /* Match the pm_runtime_disable() in device_suspend(). */ + pm_runtime_enable(dev); + goto Complete; + } + + if (!dpm_wait_for_superior(dev, async)) + goto Complete; + + dpm_watchdog_set(&wd, dev); device_lock(dev); /* @@ -586,9 +1081,6 @@ static int device_resume(struct device *dev, pm_message_t state, bool async) */ dev->power.is_prepared = false; - if (!dev->power.is_suspended) - goto Unlock; - if (dev->pm_domain) { info = "power domain "; callback = pm_op(&dev->pm_domain->ops, state); @@ -601,16 +1093,10 @@ static int device_resume(struct device *dev, pm_message_t state, bool async) goto Driver; } - if (dev->class) { - if (dev->class->pm) { - info = "class "; - callback = pm_op(dev->class->pm, state); - goto Driver; - } else if (dev->class->resume) { - info = "legacy class "; - callback = dev->class->resume; - goto End; - } + if (dev->class && dev->class->pm) { + info = "class "; + callback = pm_op(dev->class->pm, state); + goto Driver; } if (dev->bus) { @@ -632,36 +1118,32 @@ static int device_resume(struct device *dev, pm_message_t state, bool async) End: error = dpm_run_callback(callback, dev, state, info); - dev->power.is_suspended = false; - Unlock: device_unlock(dev); + dpm_watchdog_clear(&wd); Complete: complete_all(&dev->power.completion); TRACE_RESUME(error); - return error; + if (error) { + WRITE_ONCE(async_error, error); + dpm_save_failed_dev(dev_name(dev)); + pm_dev_err(dev, state, async ? " async" : "", error); + } + + dpm_async_resume_subordinate(dev, async_resume); } static void async_resume(void *data, async_cookie_t cookie) { - struct device *dev = (struct device *)data; - int error; + struct device *dev = data; - error = device_resume(dev, pm_transition, true); - if (error) - pm_dev_err(dev, pm_transition, " async", error); + device_resume(dev, pm_transition, true); put_device(dev); } -static bool is_async(struct device *dev) -{ - return dev->power.async_suspend && pm_async_enabled - && !pm_trace_is_enabled(); -} - /** * dpm_resume - Execute "resume" callbacks for non-sysdev devices. * @state: PM transition of the system being carried out. @@ -674,45 +1156,48 @@ void dpm_resume(pm_message_t state) struct device *dev; ktime_t starttime = ktime_get(); - might_sleep(); + trace_suspend_resume(TPS("dpm_resume"), state.event, true); - mutex_lock(&dpm_list_mtx); pm_transition = state; async_error = 0; + mutex_lock(&dpm_list_mtx); + + /* + * Start processing "async" root devices upfront so they don't wait for + * the "sync" devices they don't depend on. + */ list_for_each_entry(dev, &dpm_suspended_list, power.entry) { - INIT_COMPLETION(dev->power.completion); - if (is_async(dev)) { - get_device(dev); - async_schedule(async_resume, dev); - } + dpm_clear_async_state(dev); + if (dpm_root_device(dev)) + dpm_async_with_cleanup(dev, async_resume); } while (!list_empty(&dpm_suspended_list)) { dev = to_device(dpm_suspended_list.next); - get_device(dev); - if (!is_async(dev)) { - int error; + list_move_tail(&dev->power.entry, &dpm_prepared_list); + + if (!dpm_async_fn(dev, async_resume)) { + get_device(dev); mutex_unlock(&dpm_list_mtx); - error = device_resume(dev, state, false); - if (error) { - suspend_stats.failed_resume++; - dpm_save_failed_step(SUSPEND_RESUME); - dpm_save_failed_dev(dev_name(dev)); - pm_dev_err(dev, state, "", error); - } + device_resume(dev, state, false); + + put_device(dev); mutex_lock(&dpm_list_mtx); } - if (!list_empty(&dev->power.entry)) - list_move_tail(&dev->power.entry, &dpm_prepared_list); - put_device(dev); } mutex_unlock(&dpm_list_mtx); async_synchronize_full(); - dpm_show_time(starttime, state, NULL); + dpm_show_time(starttime, state, 0, NULL); + if (READ_ONCE(async_error)) + dpm_save_failed_step(SUSPEND_RESUME); + + cpufreq_resume(); + devfreq_resume(); + trace_suspend_resume(TPS("dpm_resume"), state.event, false); } /** @@ -723,10 +1208,10 @@ void dpm_resume(pm_message_t state) static void device_complete(struct device *dev, pm_message_t state) { void (*callback)(struct device *) = NULL; - char *info = NULL; + const char *info = NULL; if (dev->power.syscore) - return; + goto out; device_lock(dev); @@ -756,6 +1241,9 @@ static void device_complete(struct device *dev, pm_message_t state) device_unlock(dev); +out: + /* If enabling runtime PM for the device is blocked, unblock it. */ + pm_runtime_unblock(dev); pm_runtime_put(dev); } @@ -770,7 +1258,7 @@ void dpm_complete(pm_message_t state) { struct list_head list; - might_sleep(); + trace_suspend_resume(TPS("dpm_complete"), state.event, true); INIT_LIST_HEAD(&list); mutex_lock(&dpm_list_mtx); @@ -780,15 +1268,23 @@ void dpm_complete(pm_message_t state) get_device(dev); dev->power.is_prepared = false; list_move(&dev->power.entry, &list); + mutex_unlock(&dpm_list_mtx); + trace_device_pm_callback_start(dev, "", state.event); device_complete(dev, state); + trace_device_pm_callback_end(dev, 0); - mutex_lock(&dpm_list_mtx); put_device(dev); + + mutex_lock(&dpm_list_mtx); } list_splice(&list, &dpm_list); mutex_unlock(&dpm_list_mtx); + + /* Allow device probing and trigger re-probing of deferred devices */ + device_unblock_probing(); + trace_suspend_resume(TPS("dpm_complete"), state.event, false); } /** @@ -801,6 +1297,7 @@ void dpm_complete(pm_message_t state) void dpm_resume_end(pm_message_t state) { dpm_resume(state); + pm_restore_gfp_mask(); dpm_complete(state); } EXPORT_SYMBOL_GPL(dpm_resume_end); @@ -808,6 +1305,82 @@ EXPORT_SYMBOL_GPL(dpm_resume_end); /*------------------------- Suspend routines -------------------------*/ +static bool dpm_leaf_device(struct device *dev) +{ + struct device *child; + + lockdep_assert_held(&dpm_list_mtx); + + child = device_find_any_child(dev); + if (child) { + put_device(child); + + return false; + } + + /* + * Since this function is required to run under dpm_list_mtx, the + * list_empty() below will only return true if the device's list of + * consumers is actually empty before calling it. + */ + return list_empty(&dev->links.consumers); +} + +static bool dpm_async_suspend_parent(struct device *dev, async_func_t func) +{ + guard(mutex)(&dpm_list_mtx); + + /* + * If the device is suspended asynchronously and the parent's callback + * deletes both the device and the parent itself, the parent object may + * be freed while this function is running, so avoid that by checking + * if the device has been deleted already as the parent cannot be + * deleted before it. + */ + if (!device_pm_initialized(dev)) + return false; + + /* Start processing the device's parent if it is "async". */ + if (dev->parent) + dpm_async_with_cleanup(dev->parent, func); + + return true; +} + +static void dpm_async_suspend_superior(struct device *dev, async_func_t func) +{ + struct device_link *link; + int idx; + + if (!dpm_async_suspend_parent(dev, func)) + return; + + idx = device_links_read_lock(); + + /* Start processing the device's "async" suppliers. */ + dev_for_each_link_to_supplier(link, dev) + if (READ_ONCE(link->status) != DL_STATE_DORMANT) + dpm_async_with_cleanup(link->supplier, func); + + device_links_read_unlock(idx); +} + +static void dpm_async_suspend_complete_all(struct list_head *device_list) +{ + struct device *dev; + + guard(mutex)(&async_wip_mtx); + + list_for_each_entry_reverse(dev, device_list, power.entry) { + /* + * In case the device is being waited for and async processing + * has not started for it yet, let the waiters make progress. + */ + if (!dev->power.work_in_progress) + complete_all(&dev->power.completion); + } +} + /** * resume_event - Return a "resume" message for given "suspend" sleep state. * @sleep_state: PM message representing a sleep state. @@ -829,21 +1402,49 @@ static pm_message_t resume_event(pm_message_t sleep_state) return PMSG_ON; } +static void dpm_superior_set_must_resume(struct device *dev) +{ + struct device_link *link; + int idx; + + if (dev->parent) + dev->parent->power.must_resume = true; + + idx = device_links_read_lock(); + + dev_for_each_link_to_supplier(link, dev) + link->supplier->power.must_resume = true; + + device_links_read_unlock(idx); +} + +static void async_suspend_noirq(void *data, async_cookie_t cookie); + /** - * device_suspend_noirq - Execute a "late suspend" callback for given device. + * device_suspend_noirq - Execute a "noirq suspend" callback for given device. * @dev: Device to handle. * @state: PM transition of the system being carried out. + * @async: If true, the device is being suspended asynchronously. * * The driver of @dev will not receive interrupts while this function is being * executed. */ -static int device_suspend_noirq(struct device *dev, pm_message_t state) +static void device_suspend_noirq(struct device *dev, pm_message_t state, bool async) { pm_callback_t callback = NULL; - char *info = NULL; + const char *info = NULL; + int error = 0; - if (dev->power.syscore) - return 0; + TRACE_DEVICE(dev); + TRACE_SUSPEND(0); + + dpm_wait_for_subordinate(dev, async); + + if (READ_ONCE(async_error)) + goto Complete; + + if (dev->power.syscore || dev->power.direct_complete) + goto Complete; if (dev->pm_domain) { info = "noirq power domain "; @@ -858,80 +1459,201 @@ static int device_suspend_noirq(struct device *dev, pm_message_t state) info = "noirq bus "; callback = pm_noirq_op(dev->bus->pm, state); } + if (callback) + goto Run; - if (!callback && dev->driver && dev->driver->pm) { + if (dev_pm_skip_suspend(dev)) + goto Skip; + + if (dev->driver && dev->driver->pm) { info = "noirq driver "; callback = pm_noirq_op(dev->driver->pm, state); } - return dpm_run_callback(callback, dev, state, info); +Run: + error = dpm_run_callback(callback, dev, state, info); + if (error) { + WRITE_ONCE(async_error, error); + dpm_save_failed_dev(dev_name(dev)); + pm_dev_err(dev, state, async ? " async noirq" : " noirq", error); + goto Complete; + } + +Skip: + dev->power.is_noirq_suspended = true; + + /* + * Devices must be resumed unless they are explicitly allowed to be left + * in suspend, but even in that case skipping the resume of devices that + * were in use right before the system suspend (as indicated by their + * runtime PM usage counters and child counters) would be suboptimal. + */ + if (!(dev_pm_test_driver_flags(dev, DPM_FLAG_MAY_SKIP_RESUME) && + dev->power.may_skip_resume) || !pm_runtime_need_not_resume(dev)) + dev->power.must_resume = true; + + if (dev->power.must_resume) + dpm_superior_set_must_resume(dev); + +Complete: + complete_all(&dev->power.completion); + TRACE_SUSPEND(error); + + if (error || READ_ONCE(async_error)) + return; + + dpm_async_suspend_superior(dev, async_suspend_noirq); } -/** - * dpm_suspend_noirq - Execute "noirq suspend" callbacks for all devices. - * @state: PM transition of the system being carried out. - * - * Prevent device drivers from receiving interrupts and call the "noirq" suspend - * handlers for all non-sysdev devices. - */ -static int dpm_suspend_noirq(pm_message_t state) +static void async_suspend_noirq(void *data, async_cookie_t cookie) +{ + struct device *dev = data; + + device_suspend_noirq(dev, pm_transition, true); + put_device(dev); +} + +static int dpm_noirq_suspend_devices(pm_message_t state) { ktime_t starttime = ktime_get(); - int error = 0; + struct device *dev; + int error; + + trace_suspend_resume(TPS("dpm_suspend_noirq"), state.event, true); + + pm_transition = state; + async_error = 0; - cpuidle_pause(); - suspend_device_irqs(); mutex_lock(&dpm_list_mtx); + + /* + * Start processing "async" leaf devices upfront so they don't need to + * wait for the "sync" devices they don't depend on. + */ + list_for_each_entry_reverse(dev, &dpm_late_early_list, power.entry) { + dpm_clear_async_state(dev); + if (dpm_leaf_device(dev)) + dpm_async_with_cleanup(dev, async_suspend_noirq); + } + while (!list_empty(&dpm_late_early_list)) { - struct device *dev = to_device(dpm_late_early_list.prev); + dev = to_device(dpm_late_early_list.prev); + + list_move(&dev->power.entry, &dpm_noirq_list); + + if (dpm_async_fn(dev, async_suspend_noirq)) + continue; get_device(dev); + mutex_unlock(&dpm_list_mtx); - error = device_suspend_noirq(dev, state); + device_suspend_noirq(dev, state, false); - mutex_lock(&dpm_list_mtx); - if (error) { - pm_dev_err(dev, state, " noirq", error); - suspend_stats.failed_suspend_noirq++; - dpm_save_failed_step(SUSPEND_SUSPEND_NOIRQ); - dpm_save_failed_dev(dev_name(dev)); - put_device(dev); - break; - } - if (!list_empty(&dev->power.entry)) - list_move(&dev->power.entry, &dpm_noirq_list); put_device(dev); - if (pm_wakeup_pending()) { - error = -EBUSY; + mutex_lock(&dpm_list_mtx); + + if (READ_ONCE(async_error)) { + dpm_async_suspend_complete_all(&dpm_late_early_list); + /* + * Move all devices to the target list to resume them + * properly. + */ + list_splice_init(&dpm_late_early_list, &dpm_noirq_list); break; } } + mutex_unlock(&dpm_list_mtx); + + async_synchronize_full(); + + error = READ_ONCE(async_error); if (error) - dpm_resume_noirq(resume_event(state)); - else - dpm_show_time(starttime, state, "noirq"); + dpm_save_failed_step(SUSPEND_SUSPEND_NOIRQ); + + dpm_show_time(starttime, state, error, "noirq"); + trace_suspend_resume(TPS("dpm_suspend_noirq"), state.event, false); return error; } /** + * dpm_suspend_noirq - Execute "noirq suspend" callbacks for all devices. + * @state: PM transition of the system being carried out. + * + * Prevent device drivers' interrupt handlers from being called and invoke + * "noirq" suspend callbacks for all non-sysdev devices. + */ +int dpm_suspend_noirq(pm_message_t state) +{ + int ret; + + device_wakeup_arm_wake_irqs(); + suspend_device_irqs(); + + ret = dpm_noirq_suspend_devices(state); + if (ret) + dpm_resume_noirq(resume_event(state)); + + return ret; +} + +static void dpm_propagate_wakeup_to_parent(struct device *dev) +{ + struct device *parent = dev->parent; + + if (!parent) + return; + + spin_lock_irq(&parent->power.lock); + + if (device_wakeup_path(dev) && !parent->power.ignore_children) + parent->power.wakeup_path = true; + + spin_unlock_irq(&parent->power.lock); +} + +static void async_suspend_late(void *data, async_cookie_t cookie); + +/** * device_suspend_late - Execute a "late suspend" callback for given device. * @dev: Device to handle. * @state: PM transition of the system being carried out. + * @async: If true, the device is being suspended asynchronously. * * Runtime PM is disabled for @dev while this function is being executed. */ -static int device_suspend_late(struct device *dev, pm_message_t state) +static void device_suspend_late(struct device *dev, pm_message_t state, bool async) { pm_callback_t callback = NULL; - char *info = NULL; + const char *info = NULL; + int error = 0; + + TRACE_DEVICE(dev); + TRACE_SUSPEND(0); + + dpm_wait_for_subordinate(dev, async); + + if (READ_ONCE(async_error)) + goto Complete; + + if (pm_wakeup_pending()) { + WRITE_ONCE(async_error, -EBUSY); + goto Complete; + } + + if (dev->power.direct_complete) + goto Complete; + /* + * Disable runtime PM for the device without checking if there is a + * pending resume request for it. + */ __pm_runtime_disable(dev, false); if (dev->power.syscore) - return 0; + goto Skip; if (dev->pm_domain) { info = "late power domain "; @@ -946,57 +1668,118 @@ static int device_suspend_late(struct device *dev, pm_message_t state) info = "late bus "; callback = pm_late_early_op(dev->bus->pm, state); } + if (callback) + goto Run; - if (!callback && dev->driver && dev->driver->pm) { + if (dev_pm_skip_suspend(dev)) + goto Skip; + + if (dev->driver && dev->driver->pm) { info = "late driver "; callback = pm_late_early_op(dev->driver->pm, state); } - return dpm_run_callback(callback, dev, state, info); +Run: + error = dpm_run_callback(callback, dev, state, info); + if (error) { + WRITE_ONCE(async_error, error); + dpm_save_failed_dev(dev_name(dev)); + pm_dev_err(dev, state, async ? " async late" : " late", error); + pm_runtime_enable(dev); + goto Complete; + } + dpm_propagate_wakeup_to_parent(dev); + +Skip: + dev->power.is_late_suspended = true; + +Complete: + TRACE_SUSPEND(error); + complete_all(&dev->power.completion); + + if (error || READ_ONCE(async_error)) + return; + + dpm_async_suspend_superior(dev, async_suspend_late); +} + +static void async_suspend_late(void *data, async_cookie_t cookie) +{ + struct device *dev = data; + + device_suspend_late(dev, pm_transition, true); + put_device(dev); } /** * dpm_suspend_late - Execute "late suspend" callbacks for all devices. * @state: PM transition of the system being carried out. */ -static int dpm_suspend_late(pm_message_t state) +int dpm_suspend_late(pm_message_t state) { ktime_t starttime = ktime_get(); - int error = 0; + struct device *dev; + int error; + + trace_suspend_resume(TPS("dpm_suspend_late"), state.event, true); + + pm_transition = state; + async_error = 0; + + wake_up_all_idle_cpus(); mutex_lock(&dpm_list_mtx); + + /* + * Start processing "async" leaf devices upfront so they don't need to + * wait for the "sync" devices they don't depend on. + */ + list_for_each_entry_reverse(dev, &dpm_suspended_list, power.entry) { + dpm_clear_async_state(dev); + if (dpm_leaf_device(dev)) + dpm_async_with_cleanup(dev, async_suspend_late); + } + while (!list_empty(&dpm_suspended_list)) { - struct device *dev = to_device(dpm_suspended_list.prev); + dev = to_device(dpm_suspended_list.prev); + + list_move(&dev->power.entry, &dpm_late_early_list); + + if (dpm_async_fn(dev, async_suspend_late)) + continue; get_device(dev); + mutex_unlock(&dpm_list_mtx); - error = device_suspend_late(dev, state); + device_suspend_late(dev, state, false); - mutex_lock(&dpm_list_mtx); - if (error) { - pm_dev_err(dev, state, " late", error); - suspend_stats.failed_suspend_late++; - dpm_save_failed_step(SUSPEND_SUSPEND_LATE); - dpm_save_failed_dev(dev_name(dev)); - put_device(dev); - break; - } - if (!list_empty(&dev->power.entry)) - list_move(&dev->power.entry, &dpm_late_early_list); put_device(dev); - if (pm_wakeup_pending()) { - error = -EBUSY; + mutex_lock(&dpm_list_mtx); + + if (READ_ONCE(async_error)) { + dpm_async_suspend_complete_all(&dpm_suspended_list); + /* + * Move all devices to the target list to resume them + * properly. + */ + list_splice_init(&dpm_suspended_list, &dpm_late_early_list); break; } } + mutex_unlock(&dpm_list_mtx); - if (error) - dpm_resume_early(resume_event(state)); - else - dpm_show_time(starttime, state, "late"); + async_synchronize_full(); + + error = READ_ONCE(async_error); + if (error) { + dpm_save_failed_step(SUSPEND_SUSPEND_LATE); + dpm_resume_early(resume_event(state)); + } + dpm_show_time(starttime, state, error, "late"); + trace_suspend_resume(TPS("dpm_suspend_late"), state.event, false); return error; } @@ -1006,17 +1789,20 @@ static int dpm_suspend_late(pm_message_t state) */ int dpm_suspend_end(pm_message_t state) { - int error = dpm_suspend_late(state); + ktime_t starttime = ktime_get(); + int error; + + error = dpm_suspend_late(state); if (error) - return error; + goto out; error = dpm_suspend_noirq(state); - if (error) { + if (error) dpm_resume_early(resume_event(state)); - return error; - } - return 0; +out: + dpm_show_time(starttime, state, error, "end"); + return error; } EXPORT_SYMBOL_GPL(dpm_suspend_end); @@ -1025,57 +1811,118 @@ EXPORT_SYMBOL_GPL(dpm_suspend_end); * @dev: Device to suspend. * @state: PM transition of the system being carried out. * @cb: Suspend callback to execute. + * @info: string description of caller. */ static int legacy_suspend(struct device *dev, pm_message_t state, - int (*cb)(struct device *dev, pm_message_t state)) + int (*cb)(struct device *dev, pm_message_t state), + const char *info) { int error; ktime_t calltime; - calltime = initcall_debug_start(dev); + calltime = initcall_debug_start(dev, cb); + trace_device_pm_callback_start(dev, info, state.event); error = cb(dev, state); - suspend_report_result(cb, error); + trace_device_pm_callback_end(dev, error); + suspend_report_result(dev, cb, error); - initcall_debug_report(dev, calltime, error); + initcall_debug_report(dev, calltime, cb, error); return error; } +static void dpm_clear_superiors_direct_complete(struct device *dev) +{ + struct device_link *link; + int idx; + + if (dev->parent) { + spin_lock_irq(&dev->parent->power.lock); + dev->parent->power.direct_complete = false; + spin_unlock_irq(&dev->parent->power.lock); + } + + idx = device_links_read_lock(); + + dev_for_each_link_to_supplier(link, dev) { + spin_lock_irq(&link->supplier->power.lock); + link->supplier->power.direct_complete = false; + spin_unlock_irq(&link->supplier->power.lock); + } + + device_links_read_unlock(idx); +} + +static void async_suspend(void *data, async_cookie_t cookie); + /** * device_suspend - Execute "suspend" callbacks for given device. * @dev: Device to handle. * @state: PM transition of the system being carried out. * @async: If true, the device is being suspended asynchronously. */ -static int __device_suspend(struct device *dev, pm_message_t state, bool async) +static void device_suspend(struct device *dev, pm_message_t state, bool async) { pm_callback_t callback = NULL; - char *info = NULL; + const char *info = NULL; int error = 0; + DECLARE_DPM_WATCHDOG_ON_STACK(wd); - dpm_wait_for_children(dev, async); + TRACE_DEVICE(dev); + TRACE_SUSPEND(0); + + dpm_wait_for_subordinate(dev, async); - if (async_error) + if (READ_ONCE(async_error)) { + dev->power.direct_complete = false; goto Complete; + } /* - * If a device configured to wake up the system from sleep states - * has been suspended at run time and there's a resume request pending - * for it, this is equivalent to the device signaling wakeup, so the - * system suspend operation should be aborted. + * Wait for possible runtime PM transitions of the device in progress + * to complete and if there's a runtime resume request pending for it, + * resume it before proceeding with invoking the system-wide suspend + * callbacks for it. + * + * If the system-wide suspend callbacks below change the configuration + * of the device, they must disable runtime PM for it or otherwise + * ensure that its runtime-resume callbacks will not be confused by that + * change in case they are invoked going forward. */ - if (pm_runtime_barrier(dev) && device_may_wakeup(dev)) - pm_wakeup_event(dev, 0); + pm_runtime_barrier(dev); if (pm_wakeup_pending()) { - async_error = -EBUSY; + dev->power.direct_complete = false; + WRITE_ONCE(async_error, -EBUSY); goto Complete; } if (dev->power.syscore) goto Complete; + /* Avoid direct_complete to let wakeup_path propagate. */ + if (device_may_wakeup(dev) || device_wakeup_path(dev)) + dev->power.direct_complete = false; + + if (dev->power.direct_complete) { + if (pm_runtime_status_suspended(dev)) { + pm_runtime_disable(dev); + if (pm_runtime_status_suspended(dev)) { + pm_dev_dbg(dev, state, "direct-complete "); + dev->power.is_suspended = true; + goto Complete; + } + + pm_runtime_enable(dev); + } + dev->power.direct_complete = false; + } + + dev->power.may_skip_resume = true; + dev->power.must_resume = !dev_pm_test_driver_flags(dev, DPM_FLAG_MAY_SKIP_RESUME); + + dpm_watchdog_set(&wd, dev); device_lock(dev); if (dev->pm_domain) { @@ -1090,16 +1937,10 @@ static int __device_suspend(struct device *dev, pm_message_t state, bool async) goto Run; } - if (dev->class) { - if (dev->class->pm) { - info = "class "; - callback = pm_op(dev->class->pm, state); - goto Run; - } else if (dev->class->suspend) { - pm_dev_dbg(dev, state, "legacy class "); - error = legacy_suspend(dev, state, dev->class->suspend); - goto End; - } + if (dev->class && dev->class->pm) { + info = "class "; + callback = pm_op(dev->class->pm, state); + goto Run; } if (dev->bus) { @@ -1108,7 +1949,8 @@ static int __device_suspend(struct device *dev, pm_message_t state, bool async) callback = pm_op(dev->bus->pm, state); } else if (dev->bus->suspend) { pm_dev_dbg(dev, state, "legacy bus "); - error = legacy_suspend(dev, state, dev->bus->suspend); + error = legacy_suspend(dev, state, dev->bus->suspend, + "legacy bus "); goto End; } } @@ -1124,46 +1966,38 @@ static int __device_suspend(struct device *dev, pm_message_t state, bool async) End: if (!error) { dev->power.is_suspended = true; - if (dev->power.wakeup_path - && dev->parent && !dev->parent->power.ignore_children) - dev->parent->power.wakeup_path = true; + if (device_may_wakeup(dev)) + dev->power.wakeup_path = true; + + dpm_propagate_wakeup_to_parent(dev); + dpm_clear_superiors_direct_complete(dev); } device_unlock(dev); + dpm_watchdog_clear(&wd); Complete: - complete_all(&dev->power.completion); - if (error) - async_error = error; - - return error; -} - -static void async_suspend(void *data, async_cookie_t cookie) -{ - struct device *dev = (struct device *)data; - int error; - - error = __device_suspend(dev, pm_transition, true); if (error) { + WRITE_ONCE(async_error, error); dpm_save_failed_dev(dev_name(dev)); - pm_dev_err(dev, pm_transition, " async", error); + pm_dev_err(dev, state, async ? " async" : "", error); } - put_device(dev); + complete_all(&dev->power.completion); + TRACE_SUSPEND(error); + + if (error || READ_ONCE(async_error)) + return; + + dpm_async_suspend_superior(dev, async_suspend); } -static int device_suspend(struct device *dev) +static void async_suspend(void *data, async_cookie_t cookie) { - INIT_COMPLETION(dev->power.completion); + struct device *dev = data; - if (pm_async_enabled && dev->power.async_suspend) { - get_device(dev); - async_schedule(async_suspend, dev); - return 0; - } - - return __device_suspend(dev, pm_transition, false); + device_suspend(dev, pm_transition, true); + put_device(dev); } /** @@ -1173,46 +2007,112 @@ static int device_suspend(struct device *dev) int dpm_suspend(pm_message_t state) { ktime_t starttime = ktime_get(); - int error = 0; + struct device *dev; + int error; + trace_suspend_resume(TPS("dpm_suspend"), state.event, true); might_sleep(); - mutex_lock(&dpm_list_mtx); + devfreq_suspend(); + cpufreq_suspend(); + pm_transition = state; async_error = 0; + + mutex_lock(&dpm_list_mtx); + + /* + * Start processing "async" leaf devices upfront so they don't need to + * wait for the "sync" devices they don't depend on. + */ + list_for_each_entry_reverse(dev, &dpm_prepared_list, power.entry) { + dpm_clear_async_state(dev); + if (dpm_leaf_device(dev)) + dpm_async_with_cleanup(dev, async_suspend); + } + while (!list_empty(&dpm_prepared_list)) { - struct device *dev = to_device(dpm_prepared_list.prev); + dev = to_device(dpm_prepared_list.prev); + + list_move(&dev->power.entry, &dpm_suspended_list); + + if (dpm_async_fn(dev, async_suspend)) + continue; get_device(dev); + mutex_unlock(&dpm_list_mtx); - error = device_suspend(dev); + device_suspend(dev, state, false); + + put_device(dev); mutex_lock(&dpm_list_mtx); - if (error) { - pm_dev_err(dev, state, "", error); - dpm_save_failed_dev(dev_name(dev)); - put_device(dev); + + if (READ_ONCE(async_error)) { + dpm_async_suspend_complete_all(&dpm_prepared_list); + /* + * Move all devices to the target list to resume them + * properly. + */ + list_splice_init(&dpm_prepared_list, &dpm_suspended_list); break; } - if (!list_empty(&dev->power.entry)) - list_move(&dev->power.entry, &dpm_suspended_list); - put_device(dev); - if (async_error) - break; } + mutex_unlock(&dpm_list_mtx); + async_synchronize_full(); - if (!error) - error = async_error; - if (error) { - suspend_stats.failed_suspend++; + + error = READ_ONCE(async_error); + if (error) dpm_save_failed_step(SUSPEND_SUSPEND); - } else - dpm_show_time(starttime, state, NULL); + + dpm_show_time(starttime, state, error, NULL); + trace_suspend_resume(TPS("dpm_suspend"), state.event, false); return error; } +static bool device_prepare_smart_suspend(struct device *dev) +{ + struct device_link *link; + bool ret = true; + int idx; + + /* + * The "smart suspend" feature is enabled for devices whose drivers ask + * for it and for devices without PM callbacks. + * + * However, if "smart suspend" is not enabled for the device's parent + * or any of its suppliers that take runtime PM into account, it cannot + * be enabled for the device either. + */ + if (!dev->power.no_pm_callbacks && + !dev_pm_test_driver_flags(dev, DPM_FLAG_SMART_SUSPEND)) + return false; + + if (dev->parent && !dev_pm_smart_suspend(dev->parent) && + !dev->parent->power.ignore_children && !pm_runtime_blocked(dev->parent)) + return false; + + idx = device_links_read_lock(); + + dev_for_each_link_to_supplier(link, dev) { + if (!device_link_test(link, DL_FLAG_PM_RUNTIME)) + continue; + + if (!dev_pm_smart_suspend(link->supplier) && + !pm_runtime_blocked(link->supplier)) { + ret = false; + break; + } + } + + device_links_read_unlock(idx); + + return ret; +} + /** * device_prepare - Prepare a device for system power transition. * @dev: Device to handle. @@ -1224,11 +2124,8 @@ int dpm_suspend(pm_message_t state) static int device_prepare(struct device *dev, pm_message_t state) { int (*callback)(struct device *) = NULL; - char *info = NULL; - int error = 0; - - if (dev->power.syscore) - return 0; + bool smart_suspend; + int ret = 0; /* * If a device's parent goes into runtime suspend at the wrong time, @@ -1237,38 +2134,69 @@ static int device_prepare(struct device *dev, pm_message_t state) * it again during the complete phase. */ pm_runtime_get_noresume(dev); + /* + * If runtime PM is disabled for the device at this point and it has + * never been enabled so far, it should not be enabled until this system + * suspend-resume cycle is complete, so prepare to trigger a warning on + * subsequent attempts to enable it. + */ + smart_suspend = !pm_runtime_block_if_disabled(dev); + + if (dev->power.syscore) + return 0; device_lock(dev); - dev->power.wakeup_path = device_may_wakeup(dev); + dev->power.wakeup_path = false; + dev->power.out_band_wakeup = false; - if (dev->pm_domain) { - info = "preparing power domain "; + if (dev->power.no_pm_callbacks) + goto unlock; + + if (dev->pm_domain) callback = dev->pm_domain->ops.prepare; - } else if (dev->type && dev->type->pm) { - info = "preparing type "; + else if (dev->type && dev->type->pm) callback = dev->type->pm->prepare; - } else if (dev->class && dev->class->pm) { - info = "preparing class "; + else if (dev->class && dev->class->pm) callback = dev->class->pm->prepare; - } else if (dev->bus && dev->bus->pm) { - info = "preparing bus "; + else if (dev->bus && dev->bus->pm) callback = dev->bus->pm->prepare; - } - if (!callback && dev->driver && dev->driver->pm) { - info = "preparing driver "; + if (!callback && dev->driver && dev->driver->pm) callback = dev->driver->pm->prepare; - } - if (callback) { - error = callback(dev); - suspend_report_result(callback, error); - } + if (callback) + ret = callback(dev); +unlock: device_unlock(dev); - return error; + if (ret < 0) { + suspend_report_result(dev, callback, ret); + pm_runtime_put(dev); + return ret; + } + /* Do not enable "smart suspend" for devices with disabled runtime PM. */ + if (smart_suspend) + smart_suspend = device_prepare_smart_suspend(dev); + + spin_lock_irq(&dev->power.lock); + + dev->power.smart_suspend = smart_suspend; + /* + * A positive return value from ->prepare() means "this device appears + * to be runtime-suspended and its state is fine, so if it really is + * runtime-suspended, you can leave it in that state provided that you + * will do the same thing with all of its descendants". This only + * applies to suspend transitions, however. + */ + dev->power.direct_complete = state.event == PM_EVENT_SUSPEND && + (ret > 0 || dev->power.no_pm_callbacks) && + !dev_pm_test_driver_flags(dev, DPM_FLAG_NO_DIRECT_COMPLETE); + + spin_unlock_irq(&dev->power.lock); + + return 0; } /** @@ -1281,36 +2209,55 @@ int dpm_prepare(pm_message_t state) { int error = 0; - might_sleep(); + trace_suspend_resume(TPS("dpm_prepare"), state.event, true); + + /* + * Give a chance for the known devices to complete their probes, before + * disable probing of devices. This sync point is important at least + * at boot time + hibernation restore. + */ + wait_for_device_probe(); + /* + * It is unsafe if probing of devices will happen during suspend or + * hibernation and system behavior will be unpredictable in this case. + * So, let's prohibit device's probing here and defer their probes + * instead. The normal behavior will be restored in dpm_complete(). + */ + device_block_probing(); mutex_lock(&dpm_list_mtx); - while (!list_empty(&dpm_list)) { + while (!list_empty(&dpm_list) && !error) { struct device *dev = to_device(dpm_list.next); get_device(dev); + mutex_unlock(&dpm_list_mtx); + trace_device_pm_callback_start(dev, "", state.event); error = device_prepare(dev, state); + trace_device_pm_callback_end(dev, error); mutex_lock(&dpm_list_mtx); - if (error) { - if (error == -EAGAIN) { - put_device(dev); - error = 0; - continue; - } - printk(KERN_INFO "PM: Device %s not prepared " - "for power transition: code %d\n", - dev_name(dev), error); - put_device(dev); - break; + + if (!error) { + dev->power.is_prepared = true; + if (!list_empty(&dev->power.entry)) + list_move_tail(&dev->power.entry, &dpm_prepared_list); + } else if (error == -EAGAIN) { + error = 0; + } else { + dev_info(dev, "not prepared for power transition: code %d\n", + error); } - dev->power.is_prepared = true; - if (!list_empty(&dev->power.entry)) - list_move_tail(&dev->power.entry, &dpm_prepared_list); + + mutex_unlock(&dpm_list_mtx); + put_device(dev); + + mutex_lock(&dpm_list_mtx); } mutex_unlock(&dpm_list_mtx); + trace_suspend_resume(TPS("dpm_prepare"), state.event, false); return error; } @@ -1323,29 +2270,33 @@ int dpm_prepare(pm_message_t state) */ int dpm_suspend_start(pm_message_t state) { + ktime_t starttime = ktime_get(); int error; error = dpm_prepare(state); - if (error) { - suspend_stats.failed_prepare++; + if (error) dpm_save_failed_step(SUSPEND_PREPARE); - } else + else { + pm_restrict_gfp_mask(); error = dpm_suspend(state); + } + + dpm_show_time(starttime, state, error, "start"); return error; } EXPORT_SYMBOL_GPL(dpm_suspend_start); -void __suspend_report_result(const char *function, void *fn, int ret) +void __suspend_report_result(const char *function, struct device *dev, void *fn, int ret) { if (ret) - printk(KERN_ERR "%s(): %pF returns %d\n", function, fn, ret); + dev_err(dev, "%s(): %ps returns %d\n", function, fn, ret); } EXPORT_SYMBOL_GPL(__suspend_report_result); /** * device_pm_wait_for_dev - Wait for suspend/resume of a device to complete. - * @dev: Device to wait for. * @subordinate: Device that needs to wait for @dev. + * @dev: Device to wait for. */ int device_pm_wait_for_dev(struct device *subordinate, struct device *dev) { @@ -1375,3 +2326,39 @@ void dpm_for_each_dev(void *data, void (*fn)(struct device *, void *)) device_pm_unlock(); } EXPORT_SYMBOL_GPL(dpm_for_each_dev); + +static bool pm_ops_is_empty(const struct dev_pm_ops *ops) +{ + if (!ops) + return true; + + return !ops->prepare && + !ops->suspend && + !ops->suspend_late && + !ops->suspend_noirq && + !ops->resume_noirq && + !ops->resume_early && + !ops->resume && + !ops->complete; +} + +void device_pm_check_callbacks(struct device *dev) +{ + unsigned long flags; + + spin_lock_irqsave(&dev->power.lock, flags); + dev->power.no_pm_callbacks = + (!dev->bus || (pm_ops_is_empty(dev->bus->pm) && + !dev->bus->suspend && !dev->bus->resume)) && + (!dev->class || pm_ops_is_empty(dev->class->pm)) && + (!dev->type || pm_ops_is_empty(dev->type->pm)) && + (!dev->pm_domain || pm_ops_is_empty(&dev->pm_domain->ops)) && + (!dev->driver || (pm_ops_is_empty(dev->driver->pm) && + !dev->driver->suspend && !dev->driver->resume)); + spin_unlock_irqrestore(&dev->power.lock, flags); +} + +bool dev_pm_skip_suspend(struct device *dev) +{ + return dev_pm_smart_suspend(dev) && pm_runtime_status_suspended(dev); +} diff --git a/drivers/base/power/opp.c b/drivers/base/power/opp.c deleted file mode 100644 index c8ec186303db..000000000000 --- a/drivers/base/power/opp.c +++ /dev/null @@ -1,745 +0,0 @@ -/* - * Generic OPP Interface - * - * Copyright (C) 2009-2010 Texas Instruments Incorporated. - * Nishanth Menon - * Romit Dasgupta - * Kevin Hilman - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - */ - -#include <linux/kernel.h> -#include <linux/errno.h> -#include <linux/err.h> -#include <linux/init.h> -#include <linux/slab.h> -#include <linux/cpufreq.h> -#include <linux/device.h> -#include <linux/list.h> -#include <linux/rculist.h> -#include <linux/rcupdate.h> -#include <linux/opp.h> -#include <linux/of.h> -#include <linux/export.h> - -/* - * Internal data structure organization with the OPP layer library is as - * follows: - * dev_opp_list (root) - * |- device 1 (represents voltage domain 1) - * | |- opp 1 (availability, freq, voltage) - * | |- opp 2 .. - * ... ... - * | `- opp n .. - * |- device 2 (represents the next voltage domain) - * ... - * `- device m (represents mth voltage domain) - * device 1, 2.. are represented by dev_opp structure while each opp - * is represented by the opp structure. - */ - -/** - * struct opp - Generic OPP description structure - * @node: opp list node. The nodes are maintained throughout the lifetime - * of boot. It is expected only an optimal set of OPPs are - * added to the library by the SoC framework. - * RCU usage: opp list is traversed with RCU locks. node - * modification is possible realtime, hence the modifications - * are protected by the dev_opp_list_lock for integrity. - * IMPORTANT: the opp nodes should be maintained in increasing - * order. - * @available: true/false - marks if this OPP as available or not - * @rate: Frequency in hertz - * @u_volt: Nominal voltage in microvolts corresponding to this OPP - * @dev_opp: points back to the device_opp struct this opp belongs to - * @head: RCU callback head used for deferred freeing - * - * This structure stores the OPP information for a given device. - */ -struct opp { - struct list_head node; - - bool available; - unsigned long rate; - unsigned long u_volt; - - struct device_opp *dev_opp; - struct rcu_head head; -}; - -/** - * struct device_opp - Device opp structure - * @node: list node - contains the devices with OPPs that - * have been registered. Nodes once added are not modified in this - * list. - * RCU usage: nodes are not modified in the list of device_opp, - * however addition is possible and is secured by dev_opp_list_lock - * @dev: device pointer - * @head: notifier head to notify the OPP availability changes. - * @opp_list: list of opps - * - * This is an internal data structure maintaining the link to opps attached to - * a device. This structure is not meant to be shared to users as it is - * meant for book keeping and private to OPP library - */ -struct device_opp { - struct list_head node; - - struct device *dev; - struct srcu_notifier_head head; - struct list_head opp_list; -}; - -/* - * The root of the list of all devices. All device_opp structures branch off - * from here, with each device_opp containing the list of opp it supports in - * various states of availability. - */ -static LIST_HEAD(dev_opp_list); -/* Lock to allow exclusive modification to the device and opp lists */ -static DEFINE_MUTEX(dev_opp_list_lock); - -/** - * find_device_opp() - find device_opp struct using device pointer - * @dev: device pointer used to lookup device OPPs - * - * Search list of device OPPs for one containing matching device. Does a RCU - * reader operation to grab the pointer needed. - * - * Returns pointer to 'struct device_opp' if found, otherwise -ENODEV or - * -EINVAL based on type of error. - * - * Locking: This function must be called under rcu_read_lock(). device_opp - * is a RCU protected pointer. This means that device_opp is valid as long - * as we are under RCU lock. - */ -static struct device_opp *find_device_opp(struct device *dev) -{ - struct device_opp *tmp_dev_opp, *dev_opp = ERR_PTR(-ENODEV); - - if (unlikely(IS_ERR_OR_NULL(dev))) { - pr_err("%s: Invalid parameters\n", __func__); - return ERR_PTR(-EINVAL); - } - - list_for_each_entry_rcu(tmp_dev_opp, &dev_opp_list, node) { - if (tmp_dev_opp->dev == dev) { - dev_opp = tmp_dev_opp; - break; - } - } - - return dev_opp; -} - -/** - * opp_get_voltage() - Gets the voltage corresponding to an available opp - * @opp: opp for which voltage has to be returned for - * - * Return voltage in micro volt corresponding to the opp, else - * return 0 - * - * Locking: This function must be called under rcu_read_lock(). opp is a rcu - * protected pointer. This means that opp which could have been fetched by - * opp_find_freq_{exact,ceil,floor} functions is valid as long as we are - * under RCU lock. The pointer returned by the opp_find_freq family must be - * used in the same section as the usage of this function with the pointer - * prior to unlocking with rcu_read_unlock() to maintain the integrity of the - * pointer. - */ -unsigned long opp_get_voltage(struct opp *opp) -{ - struct opp *tmp_opp; - unsigned long v = 0; - - tmp_opp = rcu_dereference(opp); - if (unlikely(IS_ERR_OR_NULL(tmp_opp)) || !tmp_opp->available) - pr_err("%s: Invalid parameters\n", __func__); - else - v = tmp_opp->u_volt; - - return v; -} -EXPORT_SYMBOL_GPL(opp_get_voltage); - -/** - * opp_get_freq() - Gets the frequency corresponding to an available opp - * @opp: opp for which frequency has to be returned for - * - * Return frequency in hertz corresponding to the opp, else - * return 0 - * - * Locking: This function must be called under rcu_read_lock(). opp is a rcu - * protected pointer. This means that opp which could have been fetched by - * opp_find_freq_{exact,ceil,floor} functions is valid as long as we are - * under RCU lock. The pointer returned by the opp_find_freq family must be - * used in the same section as the usage of this function with the pointer - * prior to unlocking with rcu_read_unlock() to maintain the integrity of the - * pointer. - */ -unsigned long opp_get_freq(struct opp *opp) -{ - struct opp *tmp_opp; - unsigned long f = 0; - - tmp_opp = rcu_dereference(opp); - if (unlikely(IS_ERR_OR_NULL(tmp_opp)) || !tmp_opp->available) - pr_err("%s: Invalid parameters\n", __func__); - else - f = tmp_opp->rate; - - return f; -} -EXPORT_SYMBOL_GPL(opp_get_freq); - -/** - * opp_get_opp_count() - Get number of opps available in the opp list - * @dev: device for which we do this operation - * - * This function returns the number of available opps if there are any, - * else returns 0 if none or the corresponding error value. - * - * Locking: This function must be called under rcu_read_lock(). This function - * internally references two RCU protected structures: device_opp and opp which - * are safe as long as we are under a common RCU locked section. - */ -int opp_get_opp_count(struct device *dev) -{ - struct device_opp *dev_opp; - struct opp *temp_opp; - int count = 0; - - dev_opp = find_device_opp(dev); - if (IS_ERR(dev_opp)) { - int r = PTR_ERR(dev_opp); - dev_err(dev, "%s: device OPP not found (%d)\n", __func__, r); - return r; - } - - list_for_each_entry_rcu(temp_opp, &dev_opp->opp_list, node) { - if (temp_opp->available) - count++; - } - - return count; -} -EXPORT_SYMBOL_GPL(opp_get_opp_count); - -/** - * opp_find_freq_exact() - search for an exact frequency - * @dev: device for which we do this operation - * @freq: frequency to search for - * @available: true/false - match for available opp - * - * Searches for exact match in the opp list and returns pointer to the matching - * opp if found, else returns ERR_PTR in case of error and should be handled - * using IS_ERR. Error return values can be: - * EINVAL: for bad pointer - * ERANGE: no match found for search - * ENODEV: if device not found in list of registered devices - * - * Note: available is a modifier for the search. if available=true, then the - * match is for exact matching frequency and is available in the stored OPP - * table. if false, the match is for exact frequency which is not available. - * - * This provides a mechanism to enable an opp which is not available currently - * or the opposite as well. - * - * Locking: This function must be called under rcu_read_lock(). opp is a rcu - * protected pointer. The reason for the same is that the opp pointer which is - * returned will remain valid for use with opp_get_{voltage, freq} only while - * under the locked area. The pointer returned must be used prior to unlocking - * with rcu_read_unlock() to maintain the integrity of the pointer. - */ -struct opp *opp_find_freq_exact(struct device *dev, unsigned long freq, - bool available) -{ - struct device_opp *dev_opp; - struct opp *temp_opp, *opp = ERR_PTR(-ERANGE); - - dev_opp = find_device_opp(dev); - if (IS_ERR(dev_opp)) { - int r = PTR_ERR(dev_opp); - dev_err(dev, "%s: device OPP not found (%d)\n", __func__, r); - return ERR_PTR(r); - } - - list_for_each_entry_rcu(temp_opp, &dev_opp->opp_list, node) { - if (temp_opp->available == available && - temp_opp->rate == freq) { - opp = temp_opp; - break; - } - } - - return opp; -} -EXPORT_SYMBOL_GPL(opp_find_freq_exact); - -/** - * opp_find_freq_ceil() - Search for an rounded ceil freq - * @dev: device for which we do this operation - * @freq: Start frequency - * - * Search for the matching ceil *available* OPP from a starting freq - * for a device. - * - * Returns matching *opp and refreshes *freq accordingly, else returns - * ERR_PTR in case of error and should be handled using IS_ERR. Error return - * values can be: - * EINVAL: for bad pointer - * ERANGE: no match found for search - * ENODEV: if device not found in list of registered devices - * - * Locking: This function must be called under rcu_read_lock(). opp is a rcu - * protected pointer. The reason for the same is that the opp pointer which is - * returned will remain valid for use with opp_get_{voltage, freq} only while - * under the locked area. The pointer returned must be used prior to unlocking - * with rcu_read_unlock() to maintain the integrity of the pointer. - */ -struct opp *opp_find_freq_ceil(struct device *dev, unsigned long *freq) -{ - struct device_opp *dev_opp; - struct opp *temp_opp, *opp = ERR_PTR(-ERANGE); - - if (!dev || !freq) { - dev_err(dev, "%s: Invalid argument freq=%p\n", __func__, freq); - return ERR_PTR(-EINVAL); - } - - dev_opp = find_device_opp(dev); - if (IS_ERR(dev_opp)) - return ERR_CAST(dev_opp); - - list_for_each_entry_rcu(temp_opp, &dev_opp->opp_list, node) { - if (temp_opp->available && temp_opp->rate >= *freq) { - opp = temp_opp; - *freq = opp->rate; - break; - } - } - - return opp; -} -EXPORT_SYMBOL_GPL(opp_find_freq_ceil); - -/** - * opp_find_freq_floor() - Search for a rounded floor freq - * @dev: device for which we do this operation - * @freq: Start frequency - * - * Search for the matching floor *available* OPP from a starting freq - * for a device. - * - * Returns matching *opp and refreshes *freq accordingly, else returns - * ERR_PTR in case of error and should be handled using IS_ERR. Error return - * values can be: - * EINVAL: for bad pointer - * ERANGE: no match found for search - * ENODEV: if device not found in list of registered devices - * - * Locking: This function must be called under rcu_read_lock(). opp is a rcu - * protected pointer. The reason for the same is that the opp pointer which is - * returned will remain valid for use with opp_get_{voltage, freq} only while - * under the locked area. The pointer returned must be used prior to unlocking - * with rcu_read_unlock() to maintain the integrity of the pointer. - */ -struct opp *opp_find_freq_floor(struct device *dev, unsigned long *freq) -{ - struct device_opp *dev_opp; - struct opp *temp_opp, *opp = ERR_PTR(-ERANGE); - - if (!dev || !freq) { - dev_err(dev, "%s: Invalid argument freq=%p\n", __func__, freq); - return ERR_PTR(-EINVAL); - } - - dev_opp = find_device_opp(dev); - if (IS_ERR(dev_opp)) - return ERR_CAST(dev_opp); - - list_for_each_entry_rcu(temp_opp, &dev_opp->opp_list, node) { - if (temp_opp->available) { - /* go to the next node, before choosing prev */ - if (temp_opp->rate > *freq) - break; - else - opp = temp_opp; - } - } - if (!IS_ERR(opp)) - *freq = opp->rate; - - return opp; -} -EXPORT_SYMBOL_GPL(opp_find_freq_floor); - -/** - * opp_add() - Add an OPP table from a table definitions - * @dev: device for which we do this operation - * @freq: Frequency in Hz for this OPP - * @u_volt: Voltage in uVolts for this OPP - * - * This function adds an opp definition to the opp list and returns status. - * The opp is made available by default and it can be controlled using - * opp_enable/disable functions. - * - * Locking: The internal device_opp and opp structures are RCU protected. - * Hence this function internally uses RCU updater strategy with mutex locks - * to keep the integrity of the internal data structures. Callers should ensure - * that this function is *NOT* called under RCU protection or in contexts where - * mutex cannot be locked. - */ -int opp_add(struct device *dev, unsigned long freq, unsigned long u_volt) -{ - struct device_opp *dev_opp = NULL; - struct opp *opp, *new_opp; - struct list_head *head; - - /* allocate new OPP node */ - new_opp = kzalloc(sizeof(struct opp), GFP_KERNEL); - if (!new_opp) { - dev_warn(dev, "%s: Unable to create new OPP node\n", __func__); - return -ENOMEM; - } - - /* Hold our list modification lock here */ - mutex_lock(&dev_opp_list_lock); - - /* Check for existing list for 'dev' */ - dev_opp = find_device_opp(dev); - if (IS_ERR(dev_opp)) { - /* - * Allocate a new device OPP table. In the infrequent case - * where a new device is needed to be added, we pay this - * penalty. - */ - dev_opp = kzalloc(sizeof(struct device_opp), GFP_KERNEL); - if (!dev_opp) { - mutex_unlock(&dev_opp_list_lock); - kfree(new_opp); - dev_warn(dev, - "%s: Unable to create device OPP structure\n", - __func__); - return -ENOMEM; - } - - dev_opp->dev = dev; - srcu_init_notifier_head(&dev_opp->head); - INIT_LIST_HEAD(&dev_opp->opp_list); - - /* Secure the device list modification */ - list_add_rcu(&dev_opp->node, &dev_opp_list); - } - - /* populate the opp table */ - new_opp->dev_opp = dev_opp; - new_opp->rate = freq; - new_opp->u_volt = u_volt; - new_opp->available = true; - - /* Insert new OPP in order of increasing frequency */ - head = &dev_opp->opp_list; - list_for_each_entry_rcu(opp, &dev_opp->opp_list, node) { - if (new_opp->rate < opp->rate) - break; - else - head = &opp->node; - } - - list_add_rcu(&new_opp->node, head); - mutex_unlock(&dev_opp_list_lock); - - /* - * Notify the changes in the availability of the operable - * frequency/voltage list. - */ - srcu_notifier_call_chain(&dev_opp->head, OPP_EVENT_ADD, new_opp); - return 0; -} - -/** - * opp_set_availability() - helper to set the availability of an opp - * @dev: device for which we do this operation - * @freq: OPP frequency to modify availability - * @availability_req: availability status requested for this opp - * - * Set the availability of an OPP with an RCU operation, opp_{enable,disable} - * share a common logic which is isolated here. - * - * Returns -EINVAL for bad pointers, -ENOMEM if no memory available for the - * copy operation, returns 0 if no modifcation was done OR modification was - * successful. - * - * Locking: The internal device_opp and opp structures are RCU protected. - * Hence this function internally uses RCU updater strategy with mutex locks to - * keep the integrity of the internal data structures. Callers should ensure - * that this function is *NOT* called under RCU protection or in contexts where - * mutex locking or synchronize_rcu() blocking calls cannot be used. - */ -static int opp_set_availability(struct device *dev, unsigned long freq, - bool availability_req) -{ - struct device_opp *tmp_dev_opp, *dev_opp = ERR_PTR(-ENODEV); - struct opp *new_opp, *tmp_opp, *opp = ERR_PTR(-ENODEV); - int r = 0; - - /* keep the node allocated */ - new_opp = kmalloc(sizeof(struct opp), GFP_KERNEL); - if (!new_opp) { - dev_warn(dev, "%s: Unable to create OPP\n", __func__); - return -ENOMEM; - } - - mutex_lock(&dev_opp_list_lock); - - /* Find the device_opp */ - list_for_each_entry(tmp_dev_opp, &dev_opp_list, node) { - if (dev == tmp_dev_opp->dev) { - dev_opp = tmp_dev_opp; - break; - } - } - if (IS_ERR(dev_opp)) { - r = PTR_ERR(dev_opp); - dev_warn(dev, "%s: Device OPP not found (%d)\n", __func__, r); - goto unlock; - } - - /* Do we have the frequency? */ - list_for_each_entry(tmp_opp, &dev_opp->opp_list, node) { - if (tmp_opp->rate == freq) { - opp = tmp_opp; - break; - } - } - if (IS_ERR(opp)) { - r = PTR_ERR(opp); - goto unlock; - } - - /* Is update really needed? */ - if (opp->available == availability_req) - goto unlock; - /* copy the old data over */ - *new_opp = *opp; - - /* plug in new node */ - new_opp->available = availability_req; - - list_replace_rcu(&opp->node, &new_opp->node); - mutex_unlock(&dev_opp_list_lock); - kfree_rcu(opp, head); - - /* Notify the change of the OPP availability */ - if (availability_req) - srcu_notifier_call_chain(&dev_opp->head, OPP_EVENT_ENABLE, - new_opp); - else - srcu_notifier_call_chain(&dev_opp->head, OPP_EVENT_DISABLE, - new_opp); - - return 0; - -unlock: - mutex_unlock(&dev_opp_list_lock); - kfree(new_opp); - return r; -} - -/** - * opp_enable() - Enable a specific OPP - * @dev: device for which we do this operation - * @freq: OPP frequency to enable - * - * Enables a provided opp. If the operation is valid, this returns 0, else the - * corresponding error value. It is meant to be used for users an OPP available - * after being temporarily made unavailable with opp_disable. - * - * Locking: The internal device_opp and opp structures are RCU protected. - * Hence this function indirectly uses RCU and mutex locks to keep the - * integrity of the internal data structures. Callers should ensure that - * this function is *NOT* called under RCU protection or in contexts where - * mutex locking or synchronize_rcu() blocking calls cannot be used. - */ -int opp_enable(struct device *dev, unsigned long freq) -{ - return opp_set_availability(dev, freq, true); -} -EXPORT_SYMBOL_GPL(opp_enable); - -/** - * opp_disable() - Disable a specific OPP - * @dev: device for which we do this operation - * @freq: OPP frequency to disable - * - * Disables a provided opp. If the operation is valid, this returns - * 0, else the corresponding error value. It is meant to be a temporary - * control by users to make this OPP not available until the circumstances are - * right to make it available again (with a call to opp_enable). - * - * Locking: The internal device_opp and opp structures are RCU protected. - * Hence this function indirectly uses RCU and mutex locks to keep the - * integrity of the internal data structures. Callers should ensure that - * this function is *NOT* called under RCU protection or in contexts where - * mutex locking or synchronize_rcu() blocking calls cannot be used. - */ -int opp_disable(struct device *dev, unsigned long freq) -{ - return opp_set_availability(dev, freq, false); -} -EXPORT_SYMBOL_GPL(opp_disable); - -#ifdef CONFIG_CPU_FREQ -/** - * opp_init_cpufreq_table() - create a cpufreq table for a device - * @dev: device for which we do this operation - * @table: Cpufreq table returned back to caller - * - * Generate a cpufreq table for a provided device- this assumes that the - * opp list is already initialized and ready for usage. - * - * This function allocates required memory for the cpufreq table. It is - * expected that the caller does the required maintenance such as freeing - * the table as required. - * - * Returns -EINVAL for bad pointers, -ENODEV if the device is not found, -ENOMEM - * if no memory available for the operation (table is not populated), returns 0 - * if successful and table is populated. - * - * WARNING: It is important for the callers to ensure refreshing their copy of - * the table if any of the mentioned functions have been invoked in the interim. - * - * Locking: The internal device_opp and opp structures are RCU protected. - * To simplify the logic, we pretend we are updater and hold relevant mutex here - * Callers should ensure that this function is *NOT* called under RCU protection - * or in contexts where mutex locking cannot be used. - */ -int opp_init_cpufreq_table(struct device *dev, - struct cpufreq_frequency_table **table) -{ - struct device_opp *dev_opp; - struct opp *opp; - struct cpufreq_frequency_table *freq_table; - int i = 0; - - /* Pretend as if I am an updater */ - mutex_lock(&dev_opp_list_lock); - - dev_opp = find_device_opp(dev); - if (IS_ERR(dev_opp)) { - int r = PTR_ERR(dev_opp); - mutex_unlock(&dev_opp_list_lock); - dev_err(dev, "%s: Device OPP not found (%d)\n", __func__, r); - return r; - } - - freq_table = kzalloc(sizeof(struct cpufreq_frequency_table) * - (opp_get_opp_count(dev) + 1), GFP_KERNEL); - if (!freq_table) { - mutex_unlock(&dev_opp_list_lock); - dev_warn(dev, "%s: Unable to allocate frequency table\n", - __func__); - return -ENOMEM; - } - - list_for_each_entry(opp, &dev_opp->opp_list, node) { - if (opp->available) { - freq_table[i].driver_data = i; - freq_table[i].frequency = opp->rate / 1000; - i++; - } - } - mutex_unlock(&dev_opp_list_lock); - - freq_table[i].driver_data = i; - freq_table[i].frequency = CPUFREQ_TABLE_END; - - *table = &freq_table[0]; - - return 0; -} -EXPORT_SYMBOL_GPL(opp_init_cpufreq_table); - -/** - * opp_free_cpufreq_table() - free the cpufreq table - * @dev: device for which we do this operation - * @table: table to free - * - * Free up the table allocated by opp_init_cpufreq_table - */ -void opp_free_cpufreq_table(struct device *dev, - struct cpufreq_frequency_table **table) -{ - if (!table) - return; - - kfree(*table); - *table = NULL; -} -EXPORT_SYMBOL_GPL(opp_free_cpufreq_table); -#endif /* CONFIG_CPU_FREQ */ - -/** - * opp_get_notifier() - find notifier_head of the device with opp - * @dev: device pointer used to lookup device OPPs. - */ -struct srcu_notifier_head *opp_get_notifier(struct device *dev) -{ - struct device_opp *dev_opp = find_device_opp(dev); - - if (IS_ERR(dev_opp)) - return ERR_CAST(dev_opp); /* matching type */ - - return &dev_opp->head; -} - -#ifdef CONFIG_OF -/** - * of_init_opp_table() - Initialize opp table from device tree - * @dev: device pointer used to lookup device OPPs. - * - * Register the initial OPP table with the OPP library for given device. - */ -int of_init_opp_table(struct device *dev) -{ - const struct property *prop; - const __be32 *val; - int nr; - - prop = of_find_property(dev->of_node, "operating-points", NULL); - if (!prop) - return -ENODEV; - if (!prop->value) - return -ENODATA; - - /* - * Each OPP is a set of tuples consisting of frequency and - * voltage like <freq-kHz vol-uV>. - */ - nr = prop->length / sizeof(u32); - if (nr % 2) { - dev_err(dev, "%s: Invalid OPP list\n", __func__); - return -EINVAL; - } - - val = prop->value; - while (nr) { - unsigned long freq = be32_to_cpup(val++) * 1000; - unsigned long volt = be32_to_cpup(val++); - - if (opp_add(dev, freq, volt)) { - dev_warn(dev, "%s: Failed to add OPP %ld\n", - __func__, freq); - continue; - } - nr -= 2; - } - - return 0; -} -EXPORT_SYMBOL_GPL(of_init_opp_table); -#endif diff --git a/drivers/base/power/power.h b/drivers/base/power/power.h index cfc3226ec492..922ed457db19 100644 --- a/drivers/base/power/power.h +++ b/drivers/base/power/power.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #include <linux/pm_qos.h> static inline void device_pm_init_common(struct device *dev) @@ -9,7 +10,7 @@ static inline void device_pm_init_common(struct device *dev) } } -#ifdef CONFIG_PM_RUNTIME +#ifdef CONFIG_PM static inline void pm_runtime_early_init(struct device *dev) { @@ -18,9 +19,68 @@ static inline void pm_runtime_early_init(struct device *dev) } extern void pm_runtime_init(struct device *dev); +extern void pm_runtime_reinit(struct device *dev); extern void pm_runtime_remove(struct device *dev); +extern u64 pm_runtime_active_time(struct device *dev); + +#define WAKE_IRQ_DEDICATED_ALLOCATED BIT(0) +#define WAKE_IRQ_DEDICATED_MANAGED BIT(1) +#define WAKE_IRQ_DEDICATED_REVERSE BIT(2) +#define WAKE_IRQ_DEDICATED_MASK (WAKE_IRQ_DEDICATED_ALLOCATED | \ + WAKE_IRQ_DEDICATED_MANAGED | \ + WAKE_IRQ_DEDICATED_REVERSE) +#define WAKE_IRQ_DEDICATED_ENABLED BIT(3) + +struct wake_irq { + struct device *dev; + unsigned int status; + int irq; + const char *name; +}; + +extern void dev_pm_arm_wake_irq(struct wake_irq *wirq); +extern void dev_pm_disarm_wake_irq(struct wake_irq *wirq); +extern void dev_pm_enable_wake_irq_check(struct device *dev, + bool can_change_status); +extern void dev_pm_disable_wake_irq_check(struct device *dev, bool cond_disable); +extern void dev_pm_enable_wake_irq_complete(struct device *dev); + +#ifdef CONFIG_PM_SLEEP + +extern void device_wakeup_attach_irq(struct device *dev, struct wake_irq *wakeirq); +extern void device_wakeup_detach_irq(struct device *dev); +extern void device_wakeup_arm_wake_irqs(void); +extern void device_wakeup_disarm_wake_irqs(void); + +#else + +static inline void device_wakeup_attach_irq(struct device *dev, + struct wake_irq *wakeirq) {} + +static inline void device_wakeup_detach_irq(struct device *dev) +{ +} + +#endif /* CONFIG_PM_SLEEP */ + +/* + * sysfs.c + */ + +extern int dpm_sysfs_add(struct device *dev); +extern void dpm_sysfs_remove(struct device *dev); +extern void rpm_sysfs_remove(struct device *dev); +extern int wakeup_sysfs_add(struct device *dev); +extern void wakeup_sysfs_remove(struct device *dev); +extern int pm_qos_sysfs_add_resume_latency(struct device *dev); +extern void pm_qos_sysfs_remove_resume_latency(struct device *dev); +extern int pm_qos_sysfs_add_flags(struct device *dev); +extern void pm_qos_sysfs_remove_flags(struct device *dev); +extern int pm_qos_sysfs_add_latency_tolerance(struct device *dev); +extern void pm_qos_sysfs_remove_latency_tolerance(struct device *dev); +extern int dpm_sysfs_change_owner(struct device *dev, kuid_t kuid, kgid_t kgid); -#else /* !CONFIG_PM_RUNTIME */ +#else /* CONFIG_PM */ static inline void pm_runtime_early_init(struct device *dev) { @@ -28,9 +88,15 @@ static inline void pm_runtime_early_init(struct device *dev) } static inline void pm_runtime_init(struct device *dev) {} +static inline void pm_runtime_reinit(struct device *dev) {} static inline void pm_runtime_remove(struct device *dev) {} -#endif /* !CONFIG_PM_RUNTIME */ +static inline int dpm_sysfs_add(struct device *dev) { return 0; } +static inline void dpm_sysfs_remove(struct device *dev) {} +static inline int dpm_sysfs_change_owner(struct device *dev, kuid_t kuid, + kgid_t kgid) { return 0; } + +#endif #ifdef CONFIG_PM_SLEEP @@ -51,6 +117,19 @@ extern void device_pm_remove(struct device *); extern void device_pm_move_before(struct device *, struct device *); extern void device_pm_move_after(struct device *, struct device *); extern void device_pm_move_last(struct device *); +extern void device_pm_check_callbacks(struct device *dev); + +static inline bool device_pm_initialized(struct device *dev) +{ + return dev->power.in_dpm_list; +} + +/* drivers/base/power/wakeup_stats.c */ +extern int wakeup_source_sysfs_add(struct device *parent, + struct wakeup_source *ws); +extern void wakeup_source_sysfs_remove(struct wakeup_source *ws); + +extern int pm_wakeup_source_sysfs_add(struct device *parent); #else /* !CONFIG_PM_SLEEP */ @@ -69,6 +148,18 @@ static inline void device_pm_move_after(struct device *deva, struct device *devb) {} static inline void device_pm_move_last(struct device *dev) {} +static inline void device_pm_check_callbacks(struct device *dev) {} + +static inline bool device_pm_initialized(struct device *dev) +{ + return device_is_registered(dev); +} + +static inline int pm_wakeup_source_sysfs_add(struct device *parent) +{ + return 0; +} + #endif /* !CONFIG_PM_SLEEP */ static inline void device_pm_init(struct device *dev) @@ -77,31 +168,3 @@ static inline void device_pm_init(struct device *dev) device_pm_sleep_init(dev); pm_runtime_init(dev); } - -#ifdef CONFIG_PM - -/* - * sysfs.c - */ - -extern int dpm_sysfs_add(struct device *dev); -extern void dpm_sysfs_remove(struct device *dev); -extern void rpm_sysfs_remove(struct device *dev); -extern int wakeup_sysfs_add(struct device *dev); -extern void wakeup_sysfs_remove(struct device *dev); -extern int pm_qos_sysfs_add_latency(struct device *dev); -extern void pm_qos_sysfs_remove_latency(struct device *dev); -extern int pm_qos_sysfs_add_flags(struct device *dev); -extern void pm_qos_sysfs_remove_flags(struct device *dev); - -#else /* CONFIG_PM */ - -static inline int dpm_sysfs_add(struct device *dev) { return 0; } -static inline void dpm_sysfs_remove(struct device *dev) {} -static inline void rpm_sysfs_remove(struct device *dev) {} -static inline int wakeup_sysfs_add(struct device *dev) { return 0; } -static inline void wakeup_sysfs_remove(struct device *dev) {} -static inline int pm_qos_sysfs_add(struct device *dev) { return 0; } -static inline void pm_qos_sysfs_remove(struct device *dev) {} - -#endif diff --git a/drivers/base/power/qos-test.c b/drivers/base/power/qos-test.c new file mode 100644 index 000000000000..79fc6c4418da --- /dev/null +++ b/drivers/base/power/qos-test.c @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2019 NXP + */ +#include <kunit/test.h> +#include <linux/pm_qos.h> + +/* Basic test for aggregating two "min" requests */ +static void freq_qos_test_min(struct kunit *test) +{ + struct freq_constraints qos; + struct freq_qos_request req1, req2; + int ret; + + freq_constraints_init(&qos); + memset(&req1, 0, sizeof(req1)); + memset(&req2, 0, sizeof(req2)); + + ret = freq_qos_add_request(&qos, &req1, FREQ_QOS_MIN, 1000); + KUNIT_EXPECT_EQ(test, ret, 1); + ret = freq_qos_add_request(&qos, &req2, FREQ_QOS_MIN, 2000); + KUNIT_EXPECT_EQ(test, ret, 1); + + KUNIT_EXPECT_EQ(test, freq_qos_read_value(&qos, FREQ_QOS_MIN), 2000); + + ret = freq_qos_remove_request(&req2); + KUNIT_EXPECT_EQ(test, ret, 1); + KUNIT_EXPECT_EQ(test, freq_qos_read_value(&qos, FREQ_QOS_MIN), 1000); + + ret = freq_qos_remove_request(&req1); + KUNIT_EXPECT_EQ(test, ret, 1); + KUNIT_EXPECT_EQ(test, freq_qos_read_value(&qos, FREQ_QOS_MIN), + FREQ_QOS_MIN_DEFAULT_VALUE); +} + +/* Test that requests for MAX_DEFAULT_VALUE have no effect */ +static void freq_qos_test_maxdef(struct kunit *test) +{ + struct freq_constraints qos; + struct freq_qos_request req1, req2; + int ret; + + freq_constraints_init(&qos); + memset(&req1, 0, sizeof(req1)); + memset(&req2, 0, sizeof(req2)); + KUNIT_EXPECT_EQ(test, freq_qos_read_value(&qos, FREQ_QOS_MAX), + FREQ_QOS_MAX_DEFAULT_VALUE); + + ret = freq_qos_add_request(&qos, &req1, FREQ_QOS_MAX, + FREQ_QOS_MAX_DEFAULT_VALUE); + KUNIT_EXPECT_EQ(test, ret, 0); + ret = freq_qos_add_request(&qos, &req2, FREQ_QOS_MAX, + FREQ_QOS_MAX_DEFAULT_VALUE); + KUNIT_EXPECT_EQ(test, ret, 0); + + /* Add max 1000 */ + ret = freq_qos_update_request(&req1, 1000); + KUNIT_EXPECT_EQ(test, ret, 1); + KUNIT_EXPECT_EQ(test, freq_qos_read_value(&qos, FREQ_QOS_MAX), 1000); + + /* Add max 2000, no impact */ + ret = freq_qos_update_request(&req2, 2000); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, freq_qos_read_value(&qos, FREQ_QOS_MAX), 1000); + + /* Remove max 1000, new max 2000 */ + ret = freq_qos_remove_request(&req1); + KUNIT_EXPECT_EQ(test, ret, 1); + KUNIT_EXPECT_EQ(test, freq_qos_read_value(&qos, FREQ_QOS_MAX), 2000); +} + +/* + * Test that a freq_qos_request can be added again after removal + * + * This issue was solved by commit 05ff1ba412fd ("PM: QoS: Invalidate frequency + * QoS requests after removal") + */ +static void freq_qos_test_readd(struct kunit *test) +{ + struct freq_constraints qos; + struct freq_qos_request req; + int ret; + + freq_constraints_init(&qos); + memset(&req, 0, sizeof(req)); + KUNIT_EXPECT_EQ(test, freq_qos_read_value(&qos, FREQ_QOS_MIN), + FREQ_QOS_MIN_DEFAULT_VALUE); + + /* Add */ + ret = freq_qos_add_request(&qos, &req, FREQ_QOS_MIN, 1000); + KUNIT_EXPECT_EQ(test, ret, 1); + KUNIT_EXPECT_EQ(test, freq_qos_read_value(&qos, FREQ_QOS_MIN), 1000); + + /* Remove */ + ret = freq_qos_remove_request(&req); + KUNIT_EXPECT_EQ(test, ret, 1); + KUNIT_EXPECT_EQ(test, freq_qos_read_value(&qos, FREQ_QOS_MIN), + FREQ_QOS_MIN_DEFAULT_VALUE); + + /* Add again */ + ret = freq_qos_add_request(&qos, &req, FREQ_QOS_MIN, 2000); + KUNIT_EXPECT_EQ(test, ret, 1); + KUNIT_EXPECT_EQ(test, freq_qos_read_value(&qos, FREQ_QOS_MIN), 2000); +} + +static struct kunit_case pm_qos_test_cases[] = { + KUNIT_CASE(freq_qos_test_min), + KUNIT_CASE(freq_qos_test_maxdef), + KUNIT_CASE(freq_qos_test_readd), + {}, +}; + +static struct kunit_suite pm_qos_test_module = { + .name = "qos-kunit-test", + .test_cases = pm_qos_test_cases, +}; +kunit_test_suites(&pm_qos_test_module); diff --git a/drivers/base/power/qos.c b/drivers/base/power/qos.c index 5c1361a9e5dd..ff393cba7649 100644 --- a/drivers/base/power/qos.c +++ b/drivers/base/power/qos.c @@ -1,13 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Devices PM QoS constraints management * * Copyright (C) 2011 Texas Instruments, Inc. * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * * This module exposes the interface to kernel space for specifying * per-device PM QoS dependencies. It provides infrastructure for registration * of: @@ -17,15 +13,12 @@ * * This QoS design is best effort based. Dependents register their QoS needs. * Watchers register to keep track of the current QoS needs of the system. - * Watchers can register different types of notification callbacks: - * . a per-device notification callback using the dev_pm_qos_*_notifier API. - * The notification chain data is stored in the per-device constraint - * data struct. - * . a system-wide notification callback using the dev_pm_qos_*_global_notifier - * API. The notification chain data is stored in a static variable. + * Watchers can register a per-device notification callback using the + * dev_pm_qos_*_notifier API. The notification chain data is stored in the + * per-device constraint data struct. * * Note about the per-device constraint data struct allocation: - * . The per-device constraints data struct ptr is tored into the device + * . The per-device constraints data struct ptr is stored into the device * dev_pm_info. * . To minimize the data usage by the per-device constraints, the data struct * is only allocated at the first call to dev_pm_qos_add_request. @@ -49,8 +42,6 @@ static DEFINE_MUTEX(dev_pm_qos_mtx); static DEFINE_MUTEX(dev_pm_qos_sysfs_mtx); -static BLOCKING_NOTIFIER_HEAD(dev_pm_notifiers); - /** * __dev_pm_qos_flags - Check PM QoS flags for a given device. * @dev: Device to check the PM QoS flags for. @@ -64,6 +55,8 @@ enum pm_qos_flags_status __dev_pm_qos_flags(struct device *dev, s32 mask) struct pm_qos_flags *pqf; s32 val; + lockdep_assert_held(&dev->power.lock); + if (IS_ERR_OR_NULL(qos)) return PM_QOS_FLAGS_UNDEFINED; @@ -97,32 +90,54 @@ enum pm_qos_flags_status dev_pm_qos_flags(struct device *dev, s32 mask) EXPORT_SYMBOL_GPL(dev_pm_qos_flags); /** - * __dev_pm_qos_read_value - Get PM QoS constraint for a given device. + * __dev_pm_qos_resume_latency - Get resume latency constraint for a given device. * @dev: Device to get the PM QoS constraint value for. * * This routine must be called with dev->power.lock held. */ -s32 __dev_pm_qos_read_value(struct device *dev) +s32 __dev_pm_qos_resume_latency(struct device *dev) { - return IS_ERR_OR_NULL(dev->power.qos) ? - 0 : pm_qos_read_value(&dev->power.qos->latency); + lockdep_assert_held(&dev->power.lock); + + return dev_pm_qos_raw_resume_latency(dev); } /** * dev_pm_qos_read_value - Get PM QoS constraint for a given device (locked). * @dev: Device to get the PM QoS constraint value for. + * @type: QoS request type. */ -s32 dev_pm_qos_read_value(struct device *dev) +s32 dev_pm_qos_read_value(struct device *dev, enum dev_pm_qos_req_type type) { + struct dev_pm_qos *qos = dev->power.qos; unsigned long flags; s32 ret; spin_lock_irqsave(&dev->power.lock, flags); - ret = __dev_pm_qos_read_value(dev); + + switch (type) { + case DEV_PM_QOS_RESUME_LATENCY: + ret = IS_ERR_OR_NULL(qos) ? PM_QOS_RESUME_LATENCY_NO_CONSTRAINT + : pm_qos_read_value(&qos->resume_latency); + break; + case DEV_PM_QOS_MIN_FREQUENCY: + ret = IS_ERR_OR_NULL(qos) ? PM_QOS_MIN_FREQUENCY_DEFAULT_VALUE + : freq_qos_read_value(&qos->freq, FREQ_QOS_MIN); + break; + case DEV_PM_QOS_MAX_FREQUENCY: + ret = IS_ERR_OR_NULL(qos) ? PM_QOS_MAX_FREQUENCY_DEFAULT_VALUE + : freq_qos_read_value(&qos->freq, FREQ_QOS_MAX); + break; + default: + WARN_ON(1); + ret = 0; + } + spin_unlock_irqrestore(&dev->power.lock, flags); return ret; } +EXPORT_SYMBOL_GPL(dev_pm_qos_read_value); /** * apply_constraint - Add/modify/remove device PM QoS request. @@ -131,8 +146,7 @@ s32 dev_pm_qos_read_value(struct device *dev) * @value: Value to assign to the QoS request. * * Internal function to update the constraints list using the PM QoS core - * code and if needed call the per-device and the global notification - * callbacks + * code and if needed call the per-device callbacks. */ static int apply_constraint(struct dev_pm_qos_request *req, enum pm_qos_req_action action, s32 value) @@ -141,16 +155,25 @@ static int apply_constraint(struct dev_pm_qos_request *req, int ret; switch(req->type) { - case DEV_PM_QOS_LATENCY: - ret = pm_qos_update_target(&qos->latency, &req->data.pnode, - action, value); + case DEV_PM_QOS_RESUME_LATENCY: + if (WARN_ON(action != PM_QOS_REMOVE_REQ && value < 0)) + value = 0; + + ret = pm_qos_update_target(&qos->resume_latency, + &req->data.pnode, action, value); + break; + case DEV_PM_QOS_LATENCY_TOLERANCE: + ret = pm_qos_update_target(&qos->latency_tolerance, + &req->data.pnode, action, value); if (ret) { - value = pm_qos_read_value(&qos->latency); - blocking_notifier_call_chain(&dev_pm_notifiers, - (unsigned long)value, - req); + value = pm_qos_read_value(&qos->latency_tolerance); + req->dev->power.set_latency_tolerance(req->dev, value); } break; + case DEV_PM_QOS_MIN_FREQUENCY: + case DEV_PM_QOS_MAX_FREQUENCY: + ret = freq_qos_apply(&req->data.freq, action, value); + break; case DEV_PM_QOS_FLAGS: ret = pm_qos_update_flags(&qos->flags, &req->data.flr, action, value); @@ -179,19 +202,29 @@ static int dev_pm_qos_constraints_allocate(struct device *dev) if (!qos) return -ENOMEM; - n = kzalloc(sizeof(*n), GFP_KERNEL); + n = kcalloc(3, sizeof(*n), GFP_KERNEL); if (!n) { kfree(qos); return -ENOMEM; } - BLOCKING_INIT_NOTIFIER_HEAD(n); - c = &qos->latency; + c = &qos->resume_latency; plist_head_init(&c->list); - c->target_value = PM_QOS_DEV_LAT_DEFAULT_VALUE; - c->default_value = PM_QOS_DEV_LAT_DEFAULT_VALUE; + c->target_value = PM_QOS_RESUME_LATENCY_DEFAULT_VALUE; + c->default_value = PM_QOS_RESUME_LATENCY_DEFAULT_VALUE; + c->no_constraint_value = PM_QOS_RESUME_LATENCY_NO_CONSTRAINT; c->type = PM_QOS_MIN; c->notifiers = n; + BLOCKING_INIT_NOTIFIER_HEAD(n); + + c = &qos->latency_tolerance; + plist_head_init(&c->list); + c->target_value = PM_QOS_LATENCY_TOLERANCE_DEFAULT_VALUE; + c->default_value = PM_QOS_LATENCY_TOLERANCE_DEFAULT_VALUE; + c->no_constraint_value = PM_QOS_LATENCY_TOLERANCE_NO_CONSTRAINT; + c->type = PM_QOS_MIN; + + freq_constraints_init(&qos->freq); INIT_LIST_HEAD(&qos->flags.list); @@ -224,7 +257,7 @@ void dev_pm_qos_constraints_destroy(struct device *dev) * If the device's PM QoS resume latency limit or PM QoS flags have been * exposed to user space, they have to be hidden at this point. */ - pm_qos_sysfs_remove_latency(dev); + pm_qos_sysfs_remove_resume_latency(dev); pm_qos_sysfs_remove_flags(dev); mutex_lock(&dev_pm_qos_mtx); @@ -237,7 +270,7 @@ void dev_pm_qos_constraints_destroy(struct device *dev) goto out; /* Flush the constraints lists for the device. */ - c = &qos->latency; + c = &qos->resume_latency; plist_for_each_entry_safe(req, tmp, &c->list, data.pnode) { /* * Update constraints list and call the notification @@ -246,6 +279,27 @@ void dev_pm_qos_constraints_destroy(struct device *dev) apply_constraint(req, PM_QOS_REMOVE_REQ, PM_QOS_DEFAULT_VALUE); memset(req, 0, sizeof(*req)); } + + c = &qos->latency_tolerance; + plist_for_each_entry_safe(req, tmp, &c->list, data.pnode) { + apply_constraint(req, PM_QOS_REMOVE_REQ, PM_QOS_DEFAULT_VALUE); + memset(req, 0, sizeof(*req)); + } + + c = &qos->freq.min_freq; + plist_for_each_entry_safe(req, tmp, &c->list, data.freq.pnode) { + apply_constraint(req, PM_QOS_REMOVE_REQ, + PM_QOS_MIN_FREQUENCY_DEFAULT_VALUE); + memset(req, 0, sizeof(*req)); + } + + c = &qos->freq.max_freq; + plist_for_each_entry_safe(req, tmp, &c->list, data.freq.pnode) { + apply_constraint(req, PM_QOS_REMOVE_REQ, + PM_QOS_MAX_FREQUENCY_DEFAULT_VALUE); + memset(req, 0, sizeof(*req)); + } + f = &qos->flags; list_for_each_entry_safe(req, tmp, &f->list, data.flr.node) { apply_constraint(req, PM_QOS_REMOVE_REQ, PM_QOS_DEFAULT_VALUE); @@ -256,7 +310,7 @@ void dev_pm_qos_constraints_destroy(struct device *dev) dev->power.qos = ERR_PTR(-ENODEV); spin_unlock_irq(&dev->power.lock); - kfree(c->notifiers); + kfree(qos->resume_latency.notifiers); kfree(qos); out: @@ -265,6 +319,51 @@ void dev_pm_qos_constraints_destroy(struct device *dev) mutex_unlock(&dev_pm_qos_sysfs_mtx); } +static bool dev_pm_qos_invalid_req_type(struct device *dev, + enum dev_pm_qos_req_type type) +{ + return type == DEV_PM_QOS_LATENCY_TOLERANCE && + !dev->power.set_latency_tolerance; +} + +static int __dev_pm_qos_add_request(struct device *dev, + struct dev_pm_qos_request *req, + enum dev_pm_qos_req_type type, s32 value) +{ + int ret = 0; + + if (!dev || !req || dev_pm_qos_invalid_req_type(dev, type)) + return -EINVAL; + + if (WARN(dev_pm_qos_request_active(req), + "%s() called for already added request\n", __func__)) + return -EINVAL; + + if (IS_ERR(dev->power.qos)) + ret = -ENODEV; + else if (!dev->power.qos) + ret = dev_pm_qos_constraints_allocate(dev); + + trace_dev_pm_qos_add_request(dev_name(dev), type, value); + if (ret) + return ret; + + req->dev = dev; + req->type = type; + if (req->type == DEV_PM_QOS_MIN_FREQUENCY) + ret = freq_qos_add_request(&dev->power.qos->freq, + &req->data.freq, + FREQ_QOS_MIN, value); + else if (req->type == DEV_PM_QOS_MAX_FREQUENCY) + ret = freq_qos_add_request(&dev->power.qos->freq, + &req->data.freq, + FREQ_QOS_MAX, value); + else + ret = apply_constraint(req, PM_QOS_ADD_REQ, value); + + return ret; +} + /** * dev_pm_qos_add_request - inserts new qos request into the list * @dev: target device for the constraint @@ -290,31 +389,11 @@ void dev_pm_qos_constraints_destroy(struct device *dev) int dev_pm_qos_add_request(struct device *dev, struct dev_pm_qos_request *req, enum dev_pm_qos_req_type type, s32 value) { - int ret = 0; - - if (!dev || !req) /*guard against callers passing in null */ - return -EINVAL; - - if (WARN(dev_pm_qos_request_active(req), - "%s() called for already added request\n", __func__)) - return -EINVAL; + int ret; mutex_lock(&dev_pm_qos_mtx); - - if (IS_ERR(dev->power.qos)) - ret = -ENODEV; - else if (!dev->power.qos) - ret = dev_pm_qos_constraints_allocate(dev); - - trace_dev_pm_qos_add_request(dev_name(dev), type, value); - if (!ret) { - req->dev = dev; - req->type = type; - ret = apply_constraint(req, PM_QOS_ADD_REQ, value); - } - + ret = __dev_pm_qos_add_request(dev, req, type, value); mutex_unlock(&dev_pm_qos_mtx); - return ret; } EXPORT_SYMBOL_GPL(dev_pm_qos_add_request); @@ -341,9 +420,14 @@ static int __dev_pm_qos_update_request(struct dev_pm_qos_request *req, return -ENODEV; switch(req->type) { - case DEV_PM_QOS_LATENCY: + case DEV_PM_QOS_RESUME_LATENCY: + case DEV_PM_QOS_LATENCY_TOLERANCE: curr_value = req->data.pnode.prio; break; + case DEV_PM_QOS_MIN_FREQUENCY: + case DEV_PM_QOS_MAX_FREQUENCY: + curr_value = req->data.freq.pnode.prio; + break; case DEV_PM_QOS_FLAGS: curr_value = req->data.flr.flags; break; @@ -441,6 +525,7 @@ EXPORT_SYMBOL_GPL(dev_pm_qos_remove_request); * * @dev: target device for the constraint * @notifier: notifier block managed by caller. + * @type: request type. * * Will register the notifier into a notification chain that gets called * upon changes to the target value for the device. @@ -448,7 +533,8 @@ EXPORT_SYMBOL_GPL(dev_pm_qos_remove_request); * If the device's constraints object doesn't exist when this routine is called, * it will be created (or error code will be returned if that fails). */ -int dev_pm_qos_add_notifier(struct device *dev, struct notifier_block *notifier) +int dev_pm_qos_add_notifier(struct device *dev, struct notifier_block *notifier, + enum dev_pm_qos_req_type type) { int ret = 0; @@ -459,10 +545,28 @@ int dev_pm_qos_add_notifier(struct device *dev, struct notifier_block *notifier) else if (!dev->power.qos) ret = dev_pm_qos_constraints_allocate(dev); - if (!ret) - ret = blocking_notifier_chain_register( - dev->power.qos->latency.notifiers, notifier); + if (ret) + goto unlock; + + switch (type) { + case DEV_PM_QOS_RESUME_LATENCY: + ret = blocking_notifier_chain_register(dev->power.qos->resume_latency.notifiers, + notifier); + break; + case DEV_PM_QOS_MIN_FREQUENCY: + ret = freq_qos_add_notifier(&dev->power.qos->freq, + FREQ_QOS_MIN, notifier); + break; + case DEV_PM_QOS_MAX_FREQUENCY: + ret = freq_qos_add_notifier(&dev->power.qos->freq, + FREQ_QOS_MAX, notifier); + break; + default: + WARN_ON(1); + ret = -EINVAL; + } +unlock: mutex_unlock(&dev_pm_qos_mtx); return ret; } @@ -474,76 +578,77 @@ EXPORT_SYMBOL_GPL(dev_pm_qos_add_notifier); * * @dev: target device for the constraint * @notifier: notifier block to be removed. + * @type: request type. * * Will remove the notifier from the notification chain that gets called * upon changes to the target value. */ int dev_pm_qos_remove_notifier(struct device *dev, - struct notifier_block *notifier) + struct notifier_block *notifier, + enum dev_pm_qos_req_type type) { - int retval = 0; + int ret = 0; mutex_lock(&dev_pm_qos_mtx); /* Silently return if the constraints object is not present. */ - if (!IS_ERR_OR_NULL(dev->power.qos)) - retval = blocking_notifier_chain_unregister( - dev->power.qos->latency.notifiers, - notifier); + if (IS_ERR_OR_NULL(dev->power.qos)) + goto unlock; + switch (type) { + case DEV_PM_QOS_RESUME_LATENCY: + ret = blocking_notifier_chain_unregister(dev->power.qos->resume_latency.notifiers, + notifier); + break; + case DEV_PM_QOS_MIN_FREQUENCY: + ret = freq_qos_remove_notifier(&dev->power.qos->freq, + FREQ_QOS_MIN, notifier); + break; + case DEV_PM_QOS_MAX_FREQUENCY: + ret = freq_qos_remove_notifier(&dev->power.qos->freq, + FREQ_QOS_MAX, notifier); + break; + default: + WARN_ON(1); + ret = -EINVAL; + } + +unlock: mutex_unlock(&dev_pm_qos_mtx); - return retval; + return ret; } EXPORT_SYMBOL_GPL(dev_pm_qos_remove_notifier); /** - * dev_pm_qos_add_global_notifier - sets notification entry for changes to - * target value of the PM QoS constraints for any device - * - * @notifier: notifier block managed by caller. - * - * Will register the notifier into a notification chain that gets called - * upon changes to the target value for any device. - */ -int dev_pm_qos_add_global_notifier(struct notifier_block *notifier) -{ - return blocking_notifier_chain_register(&dev_pm_notifiers, notifier); -} -EXPORT_SYMBOL_GPL(dev_pm_qos_add_global_notifier); - -/** - * dev_pm_qos_remove_global_notifier - deletes notification for changes to - * target value of PM QoS constraints for any device - * - * @notifier: notifier block to be removed. - * - * Will remove the notifier from the notification chain that gets called - * upon changes to the target value for any device. - */ -int dev_pm_qos_remove_global_notifier(struct notifier_block *notifier) -{ - return blocking_notifier_chain_unregister(&dev_pm_notifiers, notifier); -} -EXPORT_SYMBOL_GPL(dev_pm_qos_remove_global_notifier); - -/** * dev_pm_qos_add_ancestor_request - Add PM QoS request for device's ancestor. * @dev: Device whose ancestor to add the request for. * @req: Pointer to the preallocated handle. + * @type: Type of the request. * @value: Constraint latency value. */ int dev_pm_qos_add_ancestor_request(struct device *dev, - struct dev_pm_qos_request *req, s32 value) + struct dev_pm_qos_request *req, + enum dev_pm_qos_req_type type, s32 value) { struct device *ancestor = dev->parent; int ret = -ENODEV; - while (ancestor && !ancestor->power.ignore_children) - ancestor = ancestor->parent; + switch (type) { + case DEV_PM_QOS_RESUME_LATENCY: + while (ancestor && !ancestor->power.ignore_children) + ancestor = ancestor->parent; + + break; + case DEV_PM_QOS_LATENCY_TOLERANCE: + while (ancestor && !ancestor->power.set_latency_tolerance) + ancestor = ancestor->parent; + break; + default: + ancestor = NULL; + } if (ancestor) - ret = dev_pm_qos_add_request(ancestor, req, - DEV_PM_QOS_LATENCY, value); + ret = dev_pm_qos_add_request(ancestor, req, type, value); if (ret < 0) req->dev = NULL; @@ -552,21 +657,27 @@ int dev_pm_qos_add_ancestor_request(struct device *dev, } EXPORT_SYMBOL_GPL(dev_pm_qos_add_ancestor_request); -#ifdef CONFIG_PM_RUNTIME static void __dev_pm_qos_drop_user_request(struct device *dev, enum dev_pm_qos_req_type type) { struct dev_pm_qos_request *req = NULL; switch(type) { - case DEV_PM_QOS_LATENCY: - req = dev->power.qos->latency_req; - dev->power.qos->latency_req = NULL; + case DEV_PM_QOS_RESUME_LATENCY: + req = dev->power.qos->resume_latency_req; + dev->power.qos->resume_latency_req = NULL; + break; + case DEV_PM_QOS_LATENCY_TOLERANCE: + req = dev->power.qos->latency_tolerance_req; + dev->power.qos->latency_tolerance_req = NULL; break; case DEV_PM_QOS_FLAGS: req = dev->power.qos->flags_req; dev->power.qos->flags_req = NULL; break; + default: + WARN_ON(1); + return; } __dev_pm_qos_remove_request(req); kfree(req); @@ -597,7 +708,7 @@ int dev_pm_qos_expose_latency_limit(struct device *dev, s32 value) if (!req) return -ENOMEM; - ret = dev_pm_qos_add_request(dev, req, DEV_PM_QOS_LATENCY, value); + ret = dev_pm_qos_add_request(dev, req, DEV_PM_QOS_RESUME_LATENCY, value); if (ret < 0) { kfree(req); return ret; @@ -609,7 +720,7 @@ int dev_pm_qos_expose_latency_limit(struct device *dev, s32 value) if (IS_ERR_OR_NULL(dev->power.qos)) ret = -ENODEV; - else if (dev->power.qos->latency_req) + else if (dev->power.qos->resume_latency_req) ret = -EEXIST; if (ret < 0) { @@ -618,13 +729,13 @@ int dev_pm_qos_expose_latency_limit(struct device *dev, s32 value) mutex_unlock(&dev_pm_qos_mtx); goto out; } - dev->power.qos->latency_req = req; + dev->power.qos->resume_latency_req = req; mutex_unlock(&dev_pm_qos_mtx); - ret = pm_qos_sysfs_add_latency(dev); + ret = pm_qos_sysfs_add_resume_latency(dev); if (ret) - dev_pm_qos_drop_user_request(dev, DEV_PM_QOS_LATENCY); + dev_pm_qos_drop_user_request(dev, DEV_PM_QOS_RESUME_LATENCY); out: mutex_unlock(&dev_pm_qos_sysfs_mtx); @@ -634,8 +745,8 @@ EXPORT_SYMBOL_GPL(dev_pm_qos_expose_latency_limit); static void __dev_pm_qos_hide_latency_limit(struct device *dev) { - if (!IS_ERR_OR_NULL(dev->power.qos) && dev->power.qos->latency_req) - __dev_pm_qos_drop_user_request(dev, DEV_PM_QOS_LATENCY); + if (!IS_ERR_OR_NULL(dev->power.qos) && dev->power.qos->resume_latency_req) + __dev_pm_qos_drop_user_request(dev, DEV_PM_QOS_RESUME_LATENCY); } /** @@ -646,7 +757,7 @@ void dev_pm_qos_hide_latency_limit(struct device *dev) { mutex_lock(&dev_pm_qos_sysfs_mtx); - pm_qos_sysfs_remove_latency(dev); + pm_qos_sysfs_remove_resume_latency(dev); mutex_lock(&dev_pm_qos_mtx); __dev_pm_qos_hide_latency_limit(dev); @@ -768,7 +879,105 @@ int dev_pm_qos_update_flags(struct device *dev, s32 mask, bool set) pm_runtime_put(dev); return ret; } -#else /* !CONFIG_PM_RUNTIME */ -static void __dev_pm_qos_hide_latency_limit(struct device *dev) {} -static void __dev_pm_qos_hide_flags(struct device *dev) {} -#endif /* CONFIG_PM_RUNTIME */ + +/** + * dev_pm_qos_get_user_latency_tolerance - Get user space latency tolerance. + * @dev: Device to obtain the user space latency tolerance for. + */ +s32 dev_pm_qos_get_user_latency_tolerance(struct device *dev) +{ + s32 ret; + + mutex_lock(&dev_pm_qos_mtx); + ret = IS_ERR_OR_NULL(dev->power.qos) + || !dev->power.qos->latency_tolerance_req ? + PM_QOS_LATENCY_TOLERANCE_NO_CONSTRAINT : + dev->power.qos->latency_tolerance_req->data.pnode.prio; + mutex_unlock(&dev_pm_qos_mtx); + return ret; +} + +/** + * dev_pm_qos_update_user_latency_tolerance - Update user space latency tolerance. + * @dev: Device to update the user space latency tolerance for. + * @val: New user space latency tolerance for @dev (negative values disable). + */ +int dev_pm_qos_update_user_latency_tolerance(struct device *dev, s32 val) +{ + int ret; + + mutex_lock(&dev_pm_qos_mtx); + + if (IS_ERR_OR_NULL(dev->power.qos) + || !dev->power.qos->latency_tolerance_req) { + struct dev_pm_qos_request *req; + + if (val < 0) { + if (val == PM_QOS_LATENCY_TOLERANCE_NO_CONSTRAINT) + ret = 0; + else + ret = -EINVAL; + goto out; + } + req = kzalloc(sizeof(*req), GFP_KERNEL); + if (!req) { + ret = -ENOMEM; + goto out; + } + ret = __dev_pm_qos_add_request(dev, req, DEV_PM_QOS_LATENCY_TOLERANCE, val); + if (ret < 0) { + kfree(req); + goto out; + } + dev->power.qos->latency_tolerance_req = req; + } else { + if (val < 0) { + __dev_pm_qos_drop_user_request(dev, DEV_PM_QOS_LATENCY_TOLERANCE); + ret = 0; + } else { + ret = __dev_pm_qos_update_request(dev->power.qos->latency_tolerance_req, val); + } + } + + out: + mutex_unlock(&dev_pm_qos_mtx); + return ret; +} +EXPORT_SYMBOL_GPL(dev_pm_qos_update_user_latency_tolerance); + +/** + * dev_pm_qos_expose_latency_tolerance - Expose latency tolerance to userspace + * @dev: Device whose latency tolerance to expose + */ +int dev_pm_qos_expose_latency_tolerance(struct device *dev) +{ + int ret; + + if (!dev->power.set_latency_tolerance) + return -EINVAL; + + mutex_lock(&dev_pm_qos_sysfs_mtx); + ret = pm_qos_sysfs_add_latency_tolerance(dev); + mutex_unlock(&dev_pm_qos_sysfs_mtx); + + return ret; +} +EXPORT_SYMBOL_GPL(dev_pm_qos_expose_latency_tolerance); + +/** + * dev_pm_qos_hide_latency_tolerance - Hide latency tolerance from userspace + * @dev: Device whose latency tolerance to hide + */ +void dev_pm_qos_hide_latency_tolerance(struct device *dev) +{ + mutex_lock(&dev_pm_qos_sysfs_mtx); + pm_qos_sysfs_remove_latency_tolerance(dev); + mutex_unlock(&dev_pm_qos_sysfs_mtx); + + /* Remove the request from user space now */ + pm_runtime_get_sync(dev); + dev_pm_qos_update_user_latency_tolerance(dev, + PM_QOS_LATENCY_TOLERANCE_NO_CONSTRAINT); + pm_runtime_put(dev); +} +EXPORT_SYMBOL_GPL(dev_pm_qos_hide_latency_tolerance); diff --git a/drivers/base/power/runtime-test.c b/drivers/base/power/runtime-test.c new file mode 100644 index 000000000000..1535ad2b0264 --- /dev/null +++ b/drivers/base/power/runtime-test.c @@ -0,0 +1,249 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2025 Google, Inc. + */ + +#include <linux/cleanup.h> +#include <linux/pm_runtime.h> +#include <kunit/device.h> +#include <kunit/test.h> + +#define DEVICE_NAME "pm_runtime_test_device" + +static void pm_runtime_depth_test(struct kunit *test) +{ + struct device *dev = kunit_device_register(test, DEVICE_NAME); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + + pm_runtime_enable(dev); + + KUNIT_EXPECT_TRUE(test, pm_runtime_suspended(dev)); + KUNIT_EXPECT_EQ(test, 0, pm_runtime_get_sync(dev)); + KUNIT_EXPECT_TRUE(test, pm_runtime_active(dev)); + KUNIT_EXPECT_EQ(test, 1, pm_runtime_get_sync(dev)); /* "already active" */ + KUNIT_EXPECT_EQ(test, 0, pm_runtime_put_sync(dev)); + KUNIT_EXPECT_EQ(test, 0, pm_runtime_put_sync(dev)); + KUNIT_EXPECT_TRUE(test, pm_runtime_suspended(dev)); +} + +/* Test pm_runtime_put() and friends when already suspended. */ +static void pm_runtime_already_suspended_test(struct kunit *test) +{ + struct device *dev = kunit_device_register(test, DEVICE_NAME); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + + pm_runtime_enable(dev); + KUNIT_EXPECT_TRUE(test, pm_runtime_suspended(dev)); + + pm_runtime_get_noresume(dev); + KUNIT_EXPECT_EQ(test, 1, pm_runtime_put_sync(dev)); + + KUNIT_EXPECT_EQ(test, 1, pm_runtime_suspend(dev)); + KUNIT_EXPECT_EQ(test, 1, pm_runtime_autosuspend(dev)); + KUNIT_EXPECT_EQ(test, 1, pm_request_autosuspend(dev)); + + pm_runtime_get_noresume(dev); + KUNIT_EXPECT_EQ(test, 1, pm_runtime_put_sync_autosuspend(dev)); + + pm_runtime_get_noresume(dev); + pm_runtime_put_autosuspend(dev); + + /* Grab 2 refcounts */ + pm_runtime_get_noresume(dev); + pm_runtime_get_noresume(dev); + /* The first put() sees usage_count 1 */ + KUNIT_EXPECT_EQ(test, 0, pm_runtime_put_sync_autosuspend(dev)); + /* The second put() sees usage_count 0 but tells us "already suspended". */ + KUNIT_EXPECT_EQ(test, 1, pm_runtime_put_sync_autosuspend(dev)); + + /* Should have remained suspended the whole time. */ + KUNIT_EXPECT_TRUE(test, pm_runtime_suspended(dev)); +} + +static void pm_runtime_idle_test(struct kunit *test) +{ + struct device *dev = kunit_device_register(test, DEVICE_NAME); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + + pm_runtime_enable(dev); + + KUNIT_EXPECT_TRUE(test, pm_runtime_suspended(dev)); + KUNIT_EXPECT_EQ(test, 0, pm_runtime_get_sync(dev)); + KUNIT_EXPECT_TRUE(test, pm_runtime_active(dev)); + KUNIT_EXPECT_EQ(test, -EAGAIN, pm_runtime_idle(dev)); + KUNIT_EXPECT_TRUE(test, pm_runtime_active(dev)); + pm_runtime_put_noidle(dev); + KUNIT_EXPECT_TRUE(test, pm_runtime_active(dev)); + KUNIT_EXPECT_EQ(test, 0, pm_runtime_idle(dev)); + KUNIT_EXPECT_TRUE(test, pm_runtime_suspended(dev)); + KUNIT_EXPECT_EQ(test, -EAGAIN, pm_runtime_idle(dev)); + KUNIT_EXPECT_EQ(test, -EAGAIN, pm_request_idle(dev)); +} + +static void pm_runtime_disabled_test(struct kunit *test) +{ + struct device *dev = kunit_device_register(test, DEVICE_NAME); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + + /* Never called pm_runtime_enable() */ + KUNIT_EXPECT_FALSE(test, pm_runtime_enabled(dev)); + + /* "disabled" is treated as "active" */ + KUNIT_EXPECT_TRUE(test, pm_runtime_active(dev)); + KUNIT_EXPECT_FALSE(test, pm_runtime_suspended(dev)); + + /* + * Note: these "fail", but they still acquire/release refcounts, so + * keep them balanced. + */ + KUNIT_EXPECT_EQ(test, -EACCES, pm_runtime_get(dev)); + pm_runtime_put(dev); + + KUNIT_EXPECT_EQ(test, -EACCES, pm_runtime_get_sync(dev)); + KUNIT_EXPECT_EQ(test, -EACCES, pm_runtime_put_sync(dev)); + + KUNIT_EXPECT_EQ(test, -EACCES, pm_runtime_get(dev)); + pm_runtime_put_autosuspend(dev); + + KUNIT_EXPECT_EQ(test, -EACCES, pm_runtime_resume_and_get(dev)); + KUNIT_EXPECT_EQ(test, -EACCES, pm_runtime_idle(dev)); + KUNIT_EXPECT_EQ(test, -EACCES, pm_request_idle(dev)); + KUNIT_EXPECT_EQ(test, -EACCES, pm_request_resume(dev)); + KUNIT_EXPECT_EQ(test, -EACCES, pm_request_autosuspend(dev)); + KUNIT_EXPECT_EQ(test, -EACCES, pm_runtime_suspend(dev)); + KUNIT_EXPECT_EQ(test, -EACCES, pm_runtime_resume(dev)); + KUNIT_EXPECT_EQ(test, -EACCES, pm_runtime_autosuspend(dev)); + + /* Still disabled */ + KUNIT_EXPECT_TRUE(test, pm_runtime_active(dev)); + KUNIT_EXPECT_FALSE(test, pm_runtime_enabled(dev)); +} + +static void pm_runtime_error_test(struct kunit *test) +{ + struct device *dev = kunit_device_register(test, DEVICE_NAME); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + + pm_runtime_enable(dev); + KUNIT_EXPECT_TRUE(test, pm_runtime_suspended(dev)); + + /* Fake a .runtime_resume() error */ + dev->power.runtime_error = -EIO; + + /* + * Note: these "fail", but they still acquire/release refcounts, so + * keep them balanced. + */ + KUNIT_EXPECT_EQ(test, -EINVAL, pm_runtime_get(dev)); + pm_runtime_put(dev); + + KUNIT_EXPECT_EQ(test, -EINVAL, pm_runtime_get_sync(dev)); + KUNIT_EXPECT_EQ(test, -EINVAL, pm_runtime_put_sync(dev)); + + KUNIT_EXPECT_EQ(test, -EINVAL, pm_runtime_get(dev)); + pm_runtime_put_autosuspend(dev); + + KUNIT_EXPECT_EQ(test, -EINVAL, pm_runtime_get(dev)); + KUNIT_EXPECT_EQ(test, -EINVAL, pm_runtime_put_sync_autosuspend(dev)); + + KUNIT_EXPECT_EQ(test, -EINVAL, pm_runtime_resume_and_get(dev)); + KUNIT_EXPECT_EQ(test, -EINVAL, pm_runtime_idle(dev)); + KUNIT_EXPECT_EQ(test, -EINVAL, pm_request_idle(dev)); + KUNIT_EXPECT_EQ(test, -EINVAL, pm_request_resume(dev)); + KUNIT_EXPECT_EQ(test, -EINVAL, pm_request_autosuspend(dev)); + KUNIT_EXPECT_EQ(test, -EINVAL, pm_runtime_suspend(dev)); + KUNIT_EXPECT_EQ(test, -EINVAL, pm_runtime_resume(dev)); + KUNIT_EXPECT_EQ(test, -EINVAL, pm_runtime_autosuspend(dev)); + + /* Error is still pending */ + KUNIT_EXPECT_TRUE(test, pm_runtime_suspended(dev)); + KUNIT_EXPECT_EQ(test, -EIO, dev->power.runtime_error); + /* Clear error */ + KUNIT_EXPECT_EQ(test, 0, pm_runtime_set_suspended(dev)); + KUNIT_EXPECT_EQ(test, 0, dev->power.runtime_error); + /* Still suspended */ + KUNIT_EXPECT_TRUE(test, pm_runtime_suspended(dev)); + + KUNIT_EXPECT_EQ(test, 0, pm_runtime_get(dev)); + pm_runtime_barrier(dev); + pm_runtime_put(dev); + pm_runtime_suspend(dev); /* flush the put(), to suspend */ + KUNIT_EXPECT_TRUE(test, pm_runtime_suspended(dev)); + + KUNIT_EXPECT_EQ(test, 0, pm_runtime_get_sync(dev)); + KUNIT_EXPECT_EQ(test, 0, pm_runtime_put_sync(dev)); + + KUNIT_EXPECT_EQ(test, 0, pm_runtime_get_sync(dev)); + pm_runtime_put_autosuspend(dev); + + KUNIT_EXPECT_EQ(test, 0, pm_runtime_resume_and_get(dev)); + + /* + * The following should all return -EAGAIN (usage is non-zero) or 1 + * (already resumed). + */ + KUNIT_EXPECT_EQ(test, -EAGAIN, pm_runtime_idle(dev)); + KUNIT_EXPECT_EQ(test, -EAGAIN, pm_request_idle(dev)); + KUNIT_EXPECT_EQ(test, 1, pm_request_resume(dev)); + KUNIT_EXPECT_EQ(test, -EAGAIN, pm_request_autosuspend(dev)); + KUNIT_EXPECT_EQ(test, -EAGAIN, pm_runtime_suspend(dev)); + KUNIT_EXPECT_EQ(test, 1, pm_runtime_resume(dev)); + KUNIT_EXPECT_EQ(test, -EAGAIN, pm_runtime_autosuspend(dev)); + + KUNIT_EXPECT_EQ(test, 0, pm_runtime_put_sync(dev)); + + /* Suspended again */ + KUNIT_EXPECT_TRUE(test, pm_runtime_suspended(dev)); +} + +/* + * Explore a typical probe() sequence in which a device marks itself powered, + * but doesn't hold any runtime PM reference, so it suspends as soon as it goes + * idle. + */ +static void pm_runtime_probe_active_test(struct kunit *test) +{ + struct device *dev = kunit_device_register(test, DEVICE_NAME); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + + KUNIT_EXPECT_TRUE(test, pm_runtime_status_suspended(dev)); + + KUNIT_EXPECT_EQ(test, 0, pm_runtime_set_active(dev)); + KUNIT_EXPECT_TRUE(test, pm_runtime_active(dev)); + + pm_runtime_enable(dev); + KUNIT_EXPECT_TRUE(test, pm_runtime_active(dev)); + + /* Nothing to flush. We stay active. */ + pm_runtime_barrier(dev); + KUNIT_EXPECT_TRUE(test, pm_runtime_active(dev)); + + /* Ask for idle? Now we suspend. */ + KUNIT_EXPECT_EQ(test, 0, pm_runtime_idle(dev)); + KUNIT_EXPECT_TRUE(test, pm_runtime_suspended(dev)); +} + +static struct kunit_case pm_runtime_test_cases[] = { + KUNIT_CASE(pm_runtime_depth_test), + KUNIT_CASE(pm_runtime_already_suspended_test), + KUNIT_CASE(pm_runtime_idle_test), + KUNIT_CASE(pm_runtime_disabled_test), + KUNIT_CASE(pm_runtime_error_test), + KUNIT_CASE(pm_runtime_probe_active_test), + {} +}; + +static struct kunit_suite pm_runtime_test_suite = { + .name = "pm_runtime_test_cases", + .test_cases = pm_runtime_test_cases, +}; + +kunit_test_suite(pm_runtime_test_suite); +MODULE_DESCRIPTION("Runtime power management unit test suite"); +MODULE_LICENSE("GPL"); diff --git a/drivers/base/power/runtime.c b/drivers/base/power/runtime.c index 268a35097578..84676cc24221 100644 --- a/drivers/base/power/runtime.c +++ b/drivers/base/power/runtime.c @@ -1,18 +1,66 @@ +// SPDX-License-Identifier: GPL-2.0 /* * drivers/base/power/runtime.c - Helper functions for device runtime PM * * Copyright (c) 2009 Rafael J. Wysocki <rjw@sisk.pl>, Novell Inc. * Copyright (C) 2010 Alan Stern <stern@rowland.harvard.edu> - * - * This file is released under the GPLv2. */ - -#include <linux/sched.h> +#include <linux/sched/mm.h> +#include <linux/ktime.h> +#include <linux/hrtimer.h> #include <linux/export.h> #include <linux/pm_runtime.h> +#include <linux/pm_wakeirq.h> +#include <linux/rculist.h> #include <trace/events/rpm.h> + +#include "../base.h" #include "power.h" +typedef int (*pm_callback_t)(struct device *); + +static inline pm_callback_t get_callback_ptr(const void *start, size_t offset) +{ + return *(pm_callback_t *)(start + offset); +} + +static pm_callback_t __rpm_get_driver_callback(struct device *dev, + size_t cb_offset) +{ + if (dev->driver && dev->driver->pm) + return get_callback_ptr(dev->driver->pm, cb_offset); + + return NULL; +} + +static pm_callback_t __rpm_get_callback(struct device *dev, size_t cb_offset) +{ + const struct dev_pm_ops *ops; + pm_callback_t cb = NULL; + + if (dev->pm_domain) + ops = &dev->pm_domain->ops; + else if (dev->type && dev->type->pm) + ops = dev->type->pm; + else if (dev->class && dev->class->pm) + ops = dev->class->pm; + else if (dev->bus && dev->bus->pm) + ops = dev->bus->pm; + else + ops = NULL; + + if (ops) + cb = get_callback_ptr(ops, cb_offset); + + if (!cb) + cb = __rpm_get_driver_callback(dev, cb_offset); + + return cb; +} + +#define RPM_GET_CALLBACK(dev, callback) \ + __rpm_get_callback(dev, offsetof(struct dev_pm_ops, callback)) + static int rpm_resume(struct device *dev, int rpmflags); static int rpm_suspend(struct device *dev, int rpmflags); @@ -27,30 +75,67 @@ static int rpm_suspend(struct device *dev, int rpmflags); * runtime_status field is updated, to account the time in the old state * correctly. */ -void update_pm_runtime_accounting(struct device *dev) +static void update_pm_runtime_accounting(struct device *dev) { - unsigned long now = jiffies; - unsigned long delta; + u64 now, last, delta; - delta = now - dev->power.accounting_timestamp; + if (dev->power.disable_depth > 0) + return; + last = dev->power.accounting_timestamp; + + now = ktime_get_mono_fast_ns(); dev->power.accounting_timestamp = now; - if (dev->power.disable_depth > 0) + /* + * Because ktime_get_mono_fast_ns() is not monotonic during + * timekeeping updates, ensure that 'now' is after the last saved + * timestamp. + */ + if (now < last) return; + delta = now - last; + if (dev->power.runtime_status == RPM_SUSPENDED) - dev->power.suspended_jiffies += delta; + dev->power.suspended_time += delta; else - dev->power.active_jiffies += delta; + dev->power.active_time += delta; } static void __update_runtime_status(struct device *dev, enum rpm_status status) { update_pm_runtime_accounting(dev); + trace_rpm_status(dev, status); dev->power.runtime_status = status; } +static u64 rpm_get_accounted_time(struct device *dev, bool suspended) +{ + u64 time; + unsigned long flags; + + spin_lock_irqsave(&dev->power.lock, flags); + + update_pm_runtime_accounting(dev); + time = suspended ? dev->power.suspended_time : dev->power.active_time; + + spin_unlock_irqrestore(&dev->power.lock, flags); + + return time; +} + +u64 pm_runtime_active_time(struct device *dev) +{ + return rpm_get_accounted_time(dev, false); +} + +u64 pm_runtime_suspended_time(struct device *dev) +{ + return rpm_get_accounted_time(dev, true); +} +EXPORT_SYMBOL_GPL(pm_runtime_suspended_time); + /** * pm_runtime_deactivate_timer - Deactivate given device's suspend timer. * @dev: Device to handle. @@ -58,7 +143,7 @@ static void __update_runtime_status(struct device *dev, enum rpm_status status) static void pm_runtime_deactivate_timer(struct device *dev) { if (dev->power.timer_expires > 0) { - del_timer(&dev->power.suspend_timer); + hrtimer_try_to_cancel(&dev->power.suspend_timer); dev->power.timer_expires = 0; } } @@ -84,43 +169,29 @@ static void pm_runtime_cancel_pending(struct device *dev) * Compute the autosuspend-delay expiration time based on the device's * power.last_busy time. If the delay has already expired or is disabled * (negative) or the power.use_autosuspend flag isn't set, return 0. - * Otherwise return the expiration time in jiffies (adjusted to be nonzero). + * Otherwise return the expiration time in nanoseconds (adjusted to be nonzero). * * This function may be called either with or without dev->power.lock held. * Either way it can be racy, since power.last_busy may be updated at any time. */ -unsigned long pm_runtime_autosuspend_expiration(struct device *dev) +u64 pm_runtime_autosuspend_expiration(struct device *dev) { int autosuspend_delay; - long elapsed; - unsigned long last_busy; - unsigned long expires = 0; + u64 expires; if (!dev->power.use_autosuspend) - goto out; + return 0; - autosuspend_delay = ACCESS_ONCE(dev->power.autosuspend_delay); + autosuspend_delay = READ_ONCE(dev->power.autosuspend_delay); if (autosuspend_delay < 0) - goto out; - - last_busy = ACCESS_ONCE(dev->power.last_busy); - elapsed = jiffies - last_busy; - if (elapsed < 0) - goto out; /* jiffies has wrapped around. */ + return 0; - /* - * If the autosuspend_delay is >= 1 second, align the timer by rounding - * up to the nearest second. - */ - expires = last_busy + msecs_to_jiffies(autosuspend_delay); - if (autosuspend_delay >= 1000) - expires = round_jiffies(expires); - expires += !expires; - if (elapsed >= expires - last_busy) - expires = 0; /* Already expired. */ + expires = READ_ONCE(dev->power.last_busy); + expires += (u64)autosuspend_delay * NSEC_PER_MSEC; + if (expires > ktime_get_mono_fast_ns()) + return expires; /* Expires in the future */ - out: - return expires; + return 0; } EXPORT_SYMBOL_GPL(pm_runtime_autosuspend_expiration); @@ -146,7 +217,7 @@ static int dev_memalloc_noio(struct device *dev, void *data) * resume/suspend callback of any one of its ancestors(or the * block device itself), the deadlock may be triggered inside the * memory allocation since it might not complete until the block - * device becomes active and the involed page I/O finishes. The + * device becomes active and the involved page I/O finishes. The * situation is pointed out first by Alan Stern. Network device * are involved in iSCSI kind of situation. * @@ -186,8 +257,7 @@ void pm_runtime_set_memalloc_noio(struct device *dev, bool enable) * flag was set by any one of the descendants. */ if (!dev || (!enable && - device_for_each_child(dev, NULL, - dev_memalloc_noio))) + device_for_each_child(dev, NULL, dev_memalloc_noio))) break; } mutex_unlock(&dev_hotplug_mutex); @@ -206,18 +276,17 @@ static int rpm_check_suspend_allowed(struct device *dev) retval = -EINVAL; else if (dev->power.disable_depth > 0) retval = -EACCES; - else if (atomic_read(&dev->power.usage_count) > 0) + else if (atomic_read(&dev->power.usage_count)) retval = -EAGAIN; - else if (!pm_children_suspended(dev)) + else if (!dev->power.ignore_children && atomic_read(&dev->power.child_count)) retval = -EBUSY; /* Pending resume requests take precedence over suspends. */ - else if ((dev->power.deferred_resume - && dev->power.runtime_status == RPM_SUSPENDING) - || (dev->power.request_pending - && dev->power.request == RPM_REQ_RESUME)) + else if ((dev->power.deferred_resume && + dev->power.runtime_status == RPM_SUSPENDING) || + (dev->power.request_pending && dev->power.request == RPM_REQ_RESUME)) retval = -EAGAIN; - else if (__dev_pm_qos_read_value(dev) < 0) + else if (__dev_pm_qos_resume_latency(dev) == 0) retval = -EPERM; else if (dev->power.runtime_status == RPM_SUSPENDED) retval = 1; @@ -225,6 +294,78 @@ static int rpm_check_suspend_allowed(struct device *dev) return retval; } +static int rpm_get_suppliers(struct device *dev) +{ + struct device_link *link; + + list_for_each_entry_rcu(link, &dev->links.suppliers, c_node, + device_links_read_lock_held()) { + int retval; + + if (!device_link_test(link, DL_FLAG_PM_RUNTIME)) + continue; + + retval = pm_runtime_get_sync(link->supplier); + /* Ignore suppliers with disabled runtime PM. */ + if (retval < 0 && retval != -EACCES) { + pm_runtime_put_noidle(link->supplier); + return retval; + } + refcount_inc(&link->rpm_active); + } + return 0; +} + +/** + * pm_runtime_release_supplier - Drop references to device link's supplier. + * @link: Target device link. + * + * Drop all runtime PM references associated with @link to its supplier device. + */ +void pm_runtime_release_supplier(struct device_link *link) +{ + struct device *supplier = link->supplier; + + /* + * The additional power.usage_count check is a safety net in case + * the rpm_active refcount becomes saturated, in which case + * refcount_dec_not_one() would return true forever, but it is not + * strictly necessary. + */ + while (refcount_dec_not_one(&link->rpm_active) && + atomic_read(&supplier->power.usage_count) > 0) + pm_runtime_put_noidle(supplier); +} + +static void __rpm_put_suppliers(struct device *dev, bool try_to_suspend) +{ + struct device_link *link; + + list_for_each_entry_rcu(link, &dev->links.suppliers, c_node, + device_links_read_lock_held()) { + pm_runtime_release_supplier(link); + if (try_to_suspend) + pm_request_idle(link->supplier); + } +} + +static void rpm_put_suppliers(struct device *dev) +{ + __rpm_put_suppliers(dev, true); +} + +static void rpm_suspend_suppliers(struct device *dev) +{ + struct device_link *link; + int idx = device_links_read_lock(); + + list_for_each_entry_rcu(link, &dev->links.suppliers, c_node, + device_links_read_lock_held()) + pm_request_idle(link->supplier); + + device_links_read_unlock(idx); +} + /** * __rpm_callback - Run a given runtime PM callback for a given device. * @cb: Runtime PM callback to run. @@ -233,19 +374,103 @@ static int rpm_check_suspend_allowed(struct device *dev) static int __rpm_callback(int (*cb)(struct device *), struct device *dev) __releases(&dev->power.lock) __acquires(&dev->power.lock) { - int retval; + int retval = 0, idx; + bool use_links = dev->power.links_count > 0; - if (dev->power.irq_safe) + if (dev->power.irq_safe) { spin_unlock(&dev->power.lock); - else + } else { spin_unlock_irq(&dev->power.lock); - retval = cb(dev); + /* + * Resume suppliers if necessary. + * + * The device's runtime PM status cannot change until this + * routine returns, so it is safe to read the status outside of + * the lock. + */ + if (use_links && dev->power.runtime_status == RPM_RESUMING) { + idx = device_links_read_lock(); - if (dev->power.irq_safe) + retval = rpm_get_suppliers(dev); + if (retval) { + rpm_put_suppliers(dev); + goto fail; + } + + device_links_read_unlock(idx); + } + } + + if (cb) + retval = cb(dev); + + if (dev->power.irq_safe) { spin_lock(&dev->power.lock); - else + } else { + /* + * If the device is suspending and the callback has returned + * success, drop the usage counters of the suppliers that have + * been reference counted on its resume. + * + * Do that if resume fails too. + */ + if (use_links && + ((dev->power.runtime_status == RPM_SUSPENDING && !retval) || + (dev->power.runtime_status == RPM_RESUMING && retval))) { + idx = device_links_read_lock(); + + __rpm_put_suppliers(dev, false); + +fail: + device_links_read_unlock(idx); + } + spin_lock_irq(&dev->power.lock); + } + + return retval; +} + +/** + * rpm_callback - Run a given runtime PM callback for a given device. + * @cb: Runtime PM callback to run. + * @dev: Device to run the callback for. + */ +static int rpm_callback(int (*cb)(struct device *), struct device *dev) +{ + int retval; + + if (dev->power.memalloc_noio) { + unsigned int noio_flag; + + /* + * Deadlock might be caused if memory allocation with + * GFP_KERNEL happens inside runtime_suspend and + * runtime_resume callbacks of one block device's + * ancestor or the block device itself. Network + * device might be thought as part of iSCSI block + * device, so network device and its ancestor should + * be marked as memalloc_noio too. + */ + noio_flag = memalloc_noio_save(); + retval = __rpm_callback(cb, dev); + memalloc_noio_restore(noio_flag); + } else { + retval = __rpm_callback(cb, dev); + } + + /* + * Since -EACCES means that runtime PM is disabled for the given device, + * it should not be returned by runtime PM callbacks. If it is returned + * nevertheless, assume it to be a transient error and convert it to + * -EAGAIN. + */ + if (retval == -EACCES) + retval = -EAGAIN; + + if (retval != -EAGAIN && retval != -EBUSY) + dev->power.runtime_error = retval; return retval; } @@ -258,7 +483,8 @@ static int __rpm_callback(int (*cb)(struct device *), struct device *dev) * Check if the device's runtime PM status allows it to be suspended. If * another idle notification has been started earlier, return immediately. If * the RPM_ASYNC flag is set then queue an idle-notification request; otherwise - * run the ->runtime_idle() callback directly. + * run the ->runtime_idle() callback directly. If the ->runtime_idle callback + * doesn't exist or if it returns 0, call rpm_suspend with the RPM_AUTO flag. * * This function must be called under dev->power.lock with interrupts disabled. */ @@ -272,6 +498,9 @@ static int rpm_idle(struct device *dev, int rpmflags) if (retval < 0) ; /* Conditions are wrong. */ + else if ((rpmflags & RPM_GET_PUT) && retval == 1) + ; /* put() is allowed in RPM_SUSPENDED */ + /* Idle notifications are allowed only in the RPM_ACTIVE state. */ else if (dev->power.runtime_status != RPM_ACTIVE) retval = -EAGAIN; @@ -287,13 +516,17 @@ static int rpm_idle(struct device *dev, int rpmflags) /* Act as though RPM_NOWAIT is always set. */ else if (dev->power.idle_notification) retval = -EINPROGRESS; + if (retval) goto out; /* Pending requests need to be canceled. */ dev->power.request = RPM_REQ_NONE; - if (dev->power.no_callbacks) + callback = RPM_GET_CALLBACK(dev, runtime_idle); + + /* If no callback assume success. */ + if (!callback || dev->power.no_callbacks) goto out; /* Carry out an asynchronous or a synchronous idle notification. */ @@ -309,64 +542,24 @@ static int rpm_idle(struct device *dev, int rpmflags) dev->power.idle_notification = true; - if (dev->pm_domain) - callback = dev->pm_domain->ops.runtime_idle; - else if (dev->type && dev->type->pm) - callback = dev->type->pm->runtime_idle; - else if (dev->class && dev->class->pm) - callback = dev->class->pm->runtime_idle; - else if (dev->bus && dev->bus->pm) - callback = dev->bus->pm->runtime_idle; + if (dev->power.irq_safe) + spin_unlock(&dev->power.lock); else - callback = NULL; + spin_unlock_irq(&dev->power.lock); - if (!callback && dev->driver && dev->driver->pm) - callback = dev->driver->pm->runtime_idle; + retval = callback(dev); - if (callback) - retval = __rpm_callback(callback, dev); + if (dev->power.irq_safe) + spin_lock(&dev->power.lock); + else + spin_lock_irq(&dev->power.lock); dev->power.idle_notification = false; wake_up_all(&dev->power.wait_queue); out: trace_rpm_return_int(dev, _THIS_IP_, retval); - return retval ? retval : rpm_suspend(dev, rpmflags); -} - -/** - * rpm_callback - Run a given runtime PM callback for a given device. - * @cb: Runtime PM callback to run. - * @dev: Device to run the callback for. - */ -static int rpm_callback(int (*cb)(struct device *), struct device *dev) -{ - int retval; - - if (!cb) - return -ENOSYS; - - if (dev->power.memalloc_noio) { - unsigned int noio_flag; - - /* - * Deadlock might be caused if memory allocation with - * GFP_KERNEL happens inside runtime_suspend and - * runtime_resume callbacks of one block device's - * ancestor or the block device itself. Network - * device might be thought as part of iSCSI block - * device, so network device and its ancestor should - * be marked as memalloc_noio too. - */ - noio_flag = memalloc_noio_save(); - retval = __rpm_callback(cb, dev); - memalloc_noio_restore(noio_flag); - } else { - retval = __rpm_callback(cb, dev); - } - - dev->power.runtime_error = retval; - return retval != -EACCES ? retval : -EIO; + return retval ? retval : rpm_suspend(dev, rpmflags | RPM_AUTO); } /** @@ -401,21 +594,19 @@ static int rpm_suspend(struct device *dev, int rpmflags) repeat: retval = rpm_check_suspend_allowed(dev); - if (retval < 0) - ; /* Conditions are wrong. */ + goto out; /* Conditions are wrong. */ /* Synchronous suspends are not allowed in the RPM_RESUMING state. */ - else if (dev->power.runtime_status == RPM_RESUMING && - !(rpmflags & RPM_ASYNC)) + if (dev->power.runtime_status == RPM_RESUMING && !(rpmflags & RPM_ASYNC)) retval = -EAGAIN; + if (retval) goto out; /* If the autosuspend_delay time hasn't expired yet, reschedule. */ - if ((rpmflags & RPM_AUTO) - && dev->power.runtime_status != RPM_SUSPENDING) { - unsigned long expires = pm_runtime_autosuspend_expiration(dev); + if ((rpmflags & RPM_AUTO) && dev->power.runtime_status != RPM_SUSPENDING) { + u64 expires = pm_runtime_autosuspend_expiration(dev); if (expires != 0) { /* Pending requests need to be canceled. */ @@ -428,10 +619,20 @@ static int rpm_suspend(struct device *dev, int rpmflags) * expire; pm_suspend_timer_fn() will take care of the * rest. */ - if (!(dev->power.timer_expires && time_before_eq( - dev->power.timer_expires, expires))) { + if (!(dev->power.timer_expires && + dev->power.timer_expires <= expires)) { + /* + * We add a slack of 25% to gather wakeups + * without sacrificing the granularity. + */ + u64 slack = (u64)READ_ONCE(dev->power.autosuspend_delay) * + (NSEC_PER_MSEC >> 2); + dev->power.timer_expires = expires; - mod_timer(&dev->power.suspend_timer, expires); + hrtimer_start_range_ns(&dev->power.suspend_timer, + ns_to_ktime(expires), + slack, + HRTIMER_MODE_ABS); } dev->power.timer_autosuspends = 1; goto out; @@ -491,24 +692,15 @@ static int rpm_suspend(struct device *dev, int rpmflags) __update_runtime_status(dev, RPM_SUSPENDING); - if (dev->pm_domain) - callback = dev->pm_domain->ops.runtime_suspend; - else if (dev->type && dev->type->pm) - callback = dev->type->pm->runtime_suspend; - else if (dev->class && dev->class->pm) - callback = dev->class->pm->runtime_suspend; - else if (dev->bus && dev->bus->pm) - callback = dev->bus->pm->runtime_suspend; - else - callback = NULL; - - if (!callback && dev->driver && dev->driver->pm) - callback = dev->driver->pm->runtime_suspend; + callback = RPM_GET_CALLBACK(dev, runtime_suspend); + dev_pm_enable_wake_irq_check(dev, true); retval = rpm_callback(callback, dev); if (retval) goto fail; + dev_pm_enable_wake_irq_complete(dev); + no_callback: __update_runtime_status(dev, RPM_SUSPENDED); pm_runtime_deactivate_timer(dev); @@ -526,8 +718,11 @@ static int rpm_suspend(struct device *dev, int rpmflags) goto out; } + if (dev->power.irq_safe) + goto out; + /* Maybe the parent is now able to suspend. */ - if (parent && !parent->power.ignore_children && !dev->power.irq_safe) { + if (parent && !parent->power.ignore_children) { spin_unlock(&dev->power.lock); spin_lock(&parent->power.lock); @@ -536,6 +731,14 @@ static int rpm_suspend(struct device *dev, int rpmflags) spin_lock(&dev->power.lock); } + /* Maybe the suppliers are now able to suspend. */ + if (dev->power.links_count > 0) { + spin_unlock_irq(&dev->power.lock); + + rpm_suspend_suppliers(dev); + + spin_lock_irq(&dev->power.lock); + } out: trace_rpm_return_int(dev, _THIS_IP_, retval); @@ -543,25 +746,23 @@ static int rpm_suspend(struct device *dev, int rpmflags) return retval; fail: + dev_pm_disable_wake_irq_check(dev, true); __update_runtime_status(dev, RPM_ACTIVE); dev->power.deferred_resume = false; wake_up_all(&dev->power.wait_queue); - if (retval == -EAGAIN || retval == -EBUSY) { - dev->power.runtime_error = 0; + /* + * On transient errors, if the callback routine failed an autosuspend, + * and if the last_busy time has been updated so that there is a new + * autosuspend expiration time, automatically reschedule another + * autosuspend. + */ + if (!dev->power.runtime_error && (rpmflags & RPM_AUTO) && + pm_runtime_autosuspend_expiration(dev) != 0) + goto repeat; + + pm_runtime_cancel_pending(dev); - /* - * If the callback routine failed an autosuspend, and - * if the last_busy time has been updated so that there - * is a new autosuspend expiration time, automatically - * reschedule another autosuspend. - */ - if ((rpmflags & RPM_AUTO) && - pm_runtime_autosuspend_expiration(dev) != 0) - goto repeat; - } else { - pm_runtime_cancel_pending(dev); - } goto out; } @@ -592,13 +793,17 @@ static int rpm_resume(struct device *dev, int rpmflags) trace_rpm_resume(dev, rpmflags); repeat: - if (dev->power.runtime_error) + if (dev->power.runtime_error) { retval = -EINVAL; - else if (dev->power.disable_depth == 1 && dev->power.is_suspended - && dev->power.runtime_status == RPM_ACTIVE) - retval = 1; - else if (dev->power.disable_depth > 0) - retval = -EACCES; + } else if (dev->power.disable_depth > 0) { + if (dev->power.runtime_status == RPM_ACTIVE && + dev->power.last_status == RPM_ACTIVE) + retval = 1; + else if (rpmflags & RPM_TRANSPARENT) + goto out; + else + retval = -EACCES; + } if (retval) goto out; @@ -617,15 +822,18 @@ static int rpm_resume(struct device *dev, int rpmflags) goto out; } - if (dev->power.runtime_status == RPM_RESUMING - || dev->power.runtime_status == RPM_SUSPENDING) { + if (dev->power.runtime_status == RPM_RESUMING || + dev->power.runtime_status == RPM_SUSPENDING) { DEFINE_WAIT(wait); if (rpmflags & (RPM_ASYNC | RPM_NOWAIT)) { - if (dev->power.runtime_status == RPM_SUSPENDING) + if (dev->power.runtime_status == RPM_SUSPENDING) { dev->power.deferred_resume = true; - else + if (rpmflags & RPM_NOWAIT) + retval = -EINPROGRESS; + } else { retval = -EINPROGRESS; + } goto out; } @@ -642,8 +850,8 @@ static int rpm_resume(struct device *dev, int rpmflags) for (;;) { prepare_to_wait(&dev->power.wait_queue, &wait, TASK_UNINTERRUPTIBLE); - if (dev->power.runtime_status != RPM_RESUMING - && dev->power.runtime_status != RPM_SUSPENDING) + if (dev->power.runtime_status != RPM_RESUMING && + dev->power.runtime_status != RPM_SUSPENDING) break; spin_unlock_irq(&dev->power.lock); @@ -663,9 +871,9 @@ static int rpm_resume(struct device *dev, int rpmflags) */ if (dev->power.no_callbacks && !parent && dev->parent) { spin_lock_nested(&dev->parent->power.lock, SINGLE_DEPTH_NESTING); - if (dev->parent->power.disable_depth > 0 - || dev->parent->power.ignore_children - || dev->parent->power.runtime_status == RPM_ACTIVE) { + if (dev->parent->power.disable_depth > 0 || + dev->parent->power.ignore_children || + dev->parent->power.runtime_status == RPM_ACTIVE) { atomic_inc(&dev->parent->power.child_count); spin_unlock(&dev->parent->power.lock); retval = 1; @@ -694,17 +902,18 @@ static int rpm_resume(struct device *dev, int rpmflags) parent = dev->parent; if (dev->power.irq_safe) goto skip_parent; + spin_unlock(&dev->power.lock); pm_runtime_get_noresume(parent); spin_lock(&parent->power.lock); /* - * We can resume if the parent's runtime PM is disabled or it - * is set to ignore children. + * Resume the parent if it has runtime PM enabled and not been + * set to ignore its children. */ - if (!parent->power.disable_depth - && !parent->power.ignore_children) { + if (!parent->power.disable_depth && + !parent->power.ignore_children) { rpm_resume(parent, 0); if (parent->power.runtime_status != RPM_ACTIVE) retval = -EBUSY; @@ -714,6 +923,7 @@ static int rpm_resume(struct device *dev, int rpmflags) spin_lock(&dev->power.lock); if (retval) goto out; + goto repeat; } skip_parent: @@ -723,27 +933,18 @@ static int rpm_resume(struct device *dev, int rpmflags) __update_runtime_status(dev, RPM_RESUMING); - if (dev->pm_domain) - callback = dev->pm_domain->ops.runtime_resume; - else if (dev->type && dev->type->pm) - callback = dev->type->pm->runtime_resume; - else if (dev->class && dev->class->pm) - callback = dev->class->pm->runtime_resume; - else if (dev->bus && dev->bus->pm) - callback = dev->bus->pm->runtime_resume; - else - callback = NULL; - - if (!callback && dev->driver && dev->driver->pm) - callback = dev->driver->pm->runtime_resume; + callback = RPM_GET_CALLBACK(dev, runtime_resume); + dev_pm_disable_wake_irq_check(dev, false); retval = rpm_callback(callback, dev); if (retval) { __update_runtime_status(dev, RPM_SUSPENDED); pm_runtime_cancel_pending(dev); + dev_pm_enable_wake_irq_check(dev, false); } else { no_callback: __update_runtime_status(dev, RPM_ACTIVE); + pm_runtime_mark_last_busy(dev); if (parent) atomic_inc(&parent->power.child_count); } @@ -810,27 +1011,32 @@ static void pm_runtime_work(struct work_struct *work) /** * pm_suspend_timer_fn - Timer function for pm_schedule_suspend(). - * @data: Device pointer passed by pm_schedule_suspend(). + * @timer: hrtimer used by pm_schedule_suspend(). * * Check if the time is right and queue a suspend request. */ -static void pm_suspend_timer_fn(unsigned long data) +static enum hrtimer_restart pm_suspend_timer_fn(struct hrtimer *timer) { - struct device *dev = (struct device *)data; + struct device *dev = container_of(timer, struct device, power.suspend_timer); unsigned long flags; - unsigned long expires; + u64 expires; spin_lock_irqsave(&dev->power.lock, flags); expires = dev->power.timer_expires; - /* If 'expire' is after 'jiffies' we've been called too early. */ - if (expires > 0 && !time_after(expires, jiffies)) { + /* + * If 'expires' is after the current time, we've been called + * too early. + */ + if (expires > 0 && expires <= ktime_get_mono_fast_ns()) { dev->power.timer_expires = 0; rpm_suspend(dev, dev->power.timer_autosuspends ? (RPM_ASYNC | RPM_AUTO) : RPM_ASYNC); } spin_unlock_irqrestore(&dev->power.lock, flags); + + return HRTIMER_NORESTART; } /** @@ -841,6 +1047,7 @@ static void pm_suspend_timer_fn(unsigned long data) int pm_schedule_suspend(struct device *dev, unsigned int delay) { unsigned long flags; + u64 expires; int retval; spin_lock_irqsave(&dev->power.lock, flags); @@ -857,10 +1064,10 @@ int pm_schedule_suspend(struct device *dev, unsigned int delay) /* Other scheduled or pending requests need to be canceled. */ pm_runtime_cancel_pending(dev); - dev->power.timer_expires = jiffies + msecs_to_jiffies(delay); - dev->power.timer_expires += !dev->power.timer_expires; + expires = ktime_get_mono_fast_ns() + (u64)delay * NSEC_PER_MSEC; + dev->power.timer_expires = expires; dev->power.timer_autosuspends = 0; - mod_timer(&dev->power.suspend_timer, dev->power.timer_expires); + hrtimer_start(&dev->power.suspend_timer, expires, HRTIMER_MODE_ABS); out: spin_unlock_irqrestore(&dev->power.lock, flags); @@ -869,13 +1076,33 @@ int pm_schedule_suspend(struct device *dev, unsigned int delay) } EXPORT_SYMBOL_GPL(pm_schedule_suspend); +static int rpm_drop_usage_count(struct device *dev) +{ + int ret; + + ret = atomic_sub_return(1, &dev->power.usage_count); + if (ret >= 0) + return ret; + + /* + * Because rpm_resume() does not check the usage counter, it will resume + * the device even if the usage counter is 0 or negative, so it is + * sufficient to increment the usage counter here to reverse the change + * made above. + */ + atomic_inc(&dev->power.usage_count); + dev_warn(dev, "Runtime PM usage count underflow!\n"); + return -EINVAL; +} + /** * __pm_runtime_idle - Entry point for runtime idle operations. * @dev: Device to send idle notification for. * @rpmflags: Flag bits. * * If the RPM_GET_PUT flag is set, decrement the device's usage count and - * return immediately if it is larger than zero. Then carry out an idle + * return immediately if it is larger than zero (if it becomes negative, log a + * warning, increment it, and return an error). Then carry out an idle * notification, either synchronous or asynchronous. * * This routine may be called in atomic context if the RPM_ASYNC flag is set, @@ -886,13 +1113,18 @@ int __pm_runtime_idle(struct device *dev, int rpmflags) unsigned long flags; int retval; - might_sleep_if(!(rpmflags & RPM_ASYNC) && !dev->power.irq_safe); - if (rpmflags & RPM_GET_PUT) { - if (!atomic_dec_and_test(&dev->power.usage_count)) + retval = rpm_drop_usage_count(dev); + if (retval < 0) { + return retval; + } else if (retval > 0) { + trace_rpm_usage(dev, rpmflags); return 0; + } } + might_sleep_if(!(rpmflags & RPM_ASYNC) && !dev->power.irq_safe); + spin_lock_irqsave(&dev->power.lock, flags); retval = rpm_idle(dev, rpmflags); spin_unlock_irqrestore(&dev->power.lock, flags); @@ -907,7 +1139,8 @@ EXPORT_SYMBOL_GPL(__pm_runtime_idle); * @rpmflags: Flag bits. * * If the RPM_GET_PUT flag is set, decrement the device's usage count and - * return immediately if it is larger than zero. Then carry out a suspend, + * return immediately if it is larger than zero (if it becomes negative, log a + * warning, increment it, and return an error). Then carry out a suspend, * either synchronous or asynchronous. * * This routine may be called in atomic context if the RPM_ASYNC flag is set, @@ -918,13 +1151,18 @@ int __pm_runtime_suspend(struct device *dev, int rpmflags) unsigned long flags; int retval; - might_sleep_if(!(rpmflags & RPM_ASYNC) && !dev->power.irq_safe); - if (rpmflags & RPM_GET_PUT) { - if (!atomic_dec_and_test(&dev->power.usage_count)) + retval = rpm_drop_usage_count(dev); + if (retval < 0) { + return retval; + } else if (retval > 0) { + trace_rpm_usage(dev, rpmflags); return 0; + } } + might_sleep_if(!(rpmflags & RPM_ASYNC) && !dev->power.irq_safe); + spin_lock_irqsave(&dev->power.lock, flags); retval = rpm_suspend(dev, rpmflags); spin_unlock_irqrestore(&dev->power.lock, flags); @@ -949,7 +1187,8 @@ int __pm_runtime_resume(struct device *dev, int rpmflags) unsigned long flags; int retval; - might_sleep_if(!(rpmflags & RPM_ASYNC) && !dev->power.irq_safe); + might_sleep_if(!(rpmflags & RPM_ASYNC) && !dev->power.irq_safe && + dev->power.runtime_status != RPM_ACTIVE); if (rpmflags & RPM_GET_PUT) atomic_inc(&dev->power.usage_count); @@ -963,6 +1202,91 @@ int __pm_runtime_resume(struct device *dev, int rpmflags) EXPORT_SYMBOL_GPL(__pm_runtime_resume); /** + * pm_runtime_get_conditional - Conditionally bump up device usage counter. + * @dev: Device to handle. + * @ign_usage_count: Whether or not to look at the current usage counter value. + * + * Return -EINVAL if runtime PM is disabled for @dev. + * + * Otherwise, if its runtime PM status is %RPM_ACTIVE and (1) @ign_usage_count + * is set, or (2) @dev is not ignoring children and its active child count is + * nonzero, or (3) the runtime PM usage counter of @dev is not zero, increment + * the usage counter of @dev and return 1. + * + * Otherwise, return 0 without changing the usage counter. + * + * If @ign_usage_count is %true, this function can be used to prevent suspending + * the device when its runtime PM status is %RPM_ACTIVE. + * + * If @ign_usage_count is %false, this function can be used to prevent + * suspending the device when both its runtime PM status is %RPM_ACTIVE and its + * runtime PM usage counter is not zero. + * + * The caller is responsible for decrementing the runtime PM usage counter of + * @dev after this function has returned a positive value for it. + */ +static int pm_runtime_get_conditional(struct device *dev, bool ign_usage_count) +{ + unsigned long flags; + int retval; + + spin_lock_irqsave(&dev->power.lock, flags); + if (dev->power.disable_depth > 0) { + retval = -EINVAL; + } else if (dev->power.runtime_status != RPM_ACTIVE) { + retval = 0; + } else if (ign_usage_count || (!dev->power.ignore_children && + atomic_read(&dev->power.child_count) > 0)) { + retval = 1; + atomic_inc(&dev->power.usage_count); + } else { + retval = atomic_inc_not_zero(&dev->power.usage_count); + } + trace_rpm_usage(dev, 0); + spin_unlock_irqrestore(&dev->power.lock, flags); + + return retval; +} + +/** + * pm_runtime_get_if_active - Bump up runtime PM usage counter if the device is + * in active state + * @dev: Target device. + * + * Increment the runtime PM usage counter of @dev if its runtime PM status is + * %RPM_ACTIVE, in which case it returns 1. If the device is in a different + * state, 0 is returned. -EINVAL is returned if runtime PM is disabled for the + * device, in which case also the usage_count will remain unmodified. + */ +int pm_runtime_get_if_active(struct device *dev) +{ + return pm_runtime_get_conditional(dev, true); +} +EXPORT_SYMBOL_GPL(pm_runtime_get_if_active); + +/** + * pm_runtime_get_if_in_use - Conditionally bump up runtime PM usage counter. + * @dev: Target device. + * + * Increment the runtime PM usage counter of @dev if its runtime PM status is + * %RPM_ACTIVE and its runtime PM usage counter is greater than 0 or it is not + * ignoring children and its active child count is nonzero. 1 is returned in + * this case. + * + * If @dev is in a different state or it is not in use (that is, its usage + * counter is 0, or it is ignoring children, or its active child count is 0), + * 0 is returned. + * + * -EINVAL is returned if runtime PM is disabled for the device, in which case + * also the usage counter of @dev is not updated. + */ +int pm_runtime_get_if_in_use(struct device *dev) +{ + return pm_runtime_get_conditional(dev, false); +} +EXPORT_SYMBOL_GPL(pm_runtime_get_if_in_use); + +/** * __pm_runtime_set_status - Set runtime PM status of a device. * @dev: Device to handle. * @status: New runtime PM status of the device. @@ -978,12 +1302,19 @@ EXPORT_SYMBOL_GPL(__pm_runtime_resume); * and the device parent's counter of unsuspended children is modified to * reflect the new status. If the new status is RPM_SUSPENDED, an idle * notification request for the parent is submitted. + * + * If @dev has any suppliers (as reflected by device links to them), and @status + * is RPM_ACTIVE, they will be activated upfront and if the activation of one + * of them fails, the status of @dev will be changed to RPM_SUSPENDED (instead + * of the @status value) and the suppliers will be deacticated on exit. The + * error returned by the failing supplier activation will be returned in that + * case. */ int __pm_runtime_set_status(struct device *dev, unsigned int status) { struct device *parent = dev->parent; - unsigned long flags; bool notify_parent = false; + unsigned long flags; int error = 0; if (status != RPM_ACTIVE && status != RPM_SUSPENDED) @@ -991,24 +1322,45 @@ int __pm_runtime_set_status(struct device *dev, unsigned int status) spin_lock_irqsave(&dev->power.lock, flags); - if (!dev->power.runtime_error && !dev->power.disable_depth) { + /* + * Prevent PM-runtime from being enabled for the device or return an + * error if it is enabled already and working. + */ + if (dev->power.runtime_error || dev->power.disable_depth) + dev->power.disable_depth++; + else error = -EAGAIN; - goto out; + + spin_unlock_irqrestore(&dev->power.lock, flags); + + if (error) + return error; + + /* + * If the new status is RPM_ACTIVE, the suppliers can be activated + * upfront regardless of the current status, because next time + * rpm_put_suppliers() runs, the rpm_active refcounts of the links + * involved will be dropped down to one anyway. + */ + if (status == RPM_ACTIVE) { + int idx = device_links_read_lock(); + + error = rpm_get_suppliers(dev); + if (error) + status = RPM_SUSPENDED; + + device_links_read_unlock(idx); } - if (dev->power.runtime_status == status) - goto out_set; + spin_lock_irqsave(&dev->power.lock, flags); - if (status == RPM_SUSPENDED) { - /* It always is possible to set the status to 'suspended'. */ - if (parent) { - atomic_add_unless(&parent->power.child_count, -1, 0); - notify_parent = !parent->power.ignore_children; - } + if (dev->power.runtime_status == status || !parent) goto out_set; - } - if (parent) { + if (status == RPM_SUSPENDED) { + atomic_add_unless(&parent->power.child_count, -1, 0); + notify_parent = !parent->power.ignore_children; + } else { spin_lock_nested(&parent->power.lock, SINGLE_DEPTH_NESTING); /* @@ -1016,28 +1368,46 @@ int __pm_runtime_set_status(struct device *dev, unsigned int status) * not active, has runtime PM enabled and the * 'power.ignore_children' flag unset. */ - if (!parent->power.disable_depth - && !parent->power.ignore_children - && parent->power.runtime_status != RPM_ACTIVE) + if (!parent->power.disable_depth && + !parent->power.ignore_children && + parent->power.runtime_status != RPM_ACTIVE) { + dev_err(dev, "runtime PM trying to activate child device %s but parent (%s) is not active\n", + dev_name(dev), + dev_name(parent)); error = -EBUSY; - else if (dev->power.runtime_status == RPM_SUSPENDED) + } else if (dev->power.runtime_status == RPM_SUSPENDED) { atomic_inc(&parent->power.child_count); + } spin_unlock(&parent->power.lock); - if (error) + if (error) { + status = RPM_SUSPENDED; goto out; + } } out_set: __update_runtime_status(dev, status); - dev->power.runtime_error = 0; + if (!error) + dev->power.runtime_error = 0; + out: spin_unlock_irqrestore(&dev->power.lock, flags); if (notify_parent) pm_request_idle(parent); + if (status == RPM_SUSPENDED) { + int idx = device_links_read_lock(); + + rpm_put_suppliers(dev); + + device_links_read_unlock(idx); + } + + pm_runtime_enable(dev); + return error; } EXPORT_SYMBOL_GPL(__pm_runtime_set_status); @@ -1065,9 +1435,9 @@ static void __pm_runtime_barrier(struct device *dev) dev->power.request_pending = false; } - if (dev->power.runtime_status == RPM_SUSPENDING - || dev->power.runtime_status == RPM_RESUMING - || dev->power.idle_notification) { + if (dev->power.runtime_status == RPM_SUSPENDING || + dev->power.runtime_status == RPM_RESUMING || + dev->power.idle_notification) { DEFINE_WAIT(wait); /* Suspend, wake-up or idle notification in progress. */ @@ -1097,47 +1467,48 @@ static void __pm_runtime_barrier(struct device *dev) * Next, make sure that all pending requests for the device have been flushed * from pm_wq and wait for all runtime PM operations involving the device in * progress to complete. - * - * Return value: - * 1, if there was a resume request pending and the device had to be woken up, - * 0, otherwise */ -int pm_runtime_barrier(struct device *dev) +void pm_runtime_barrier(struct device *dev) { - int retval = 0; - pm_runtime_get_noresume(dev); spin_lock_irq(&dev->power.lock); if (dev->power.request_pending - && dev->power.request == RPM_REQ_RESUME) { + && dev->power.request == RPM_REQ_RESUME) rpm_resume(dev, 0); - retval = 1; - } __pm_runtime_barrier(dev); spin_unlock_irq(&dev->power.lock); pm_runtime_put_noidle(dev); - - return retval; } EXPORT_SYMBOL_GPL(pm_runtime_barrier); -/** - * __pm_runtime_disable - Disable runtime PM of a device. - * @dev: Device to handle. - * @check_resume: If set, check if there's a resume request for the device. - * - * Increment power.disable_depth for the device and if was zero previously, - * cancel all pending runtime PM requests for the device and wait for all - * operations in progress to complete. The device can be either active or - * suspended after its runtime PM has been disabled. - * - * If @check_resume is set and there's a resume request pending when - * __pm_runtime_disable() is called and power.disable_depth is zero, the - * function will wake up the device before disabling its runtime PM. - */ +bool pm_runtime_block_if_disabled(struct device *dev) +{ + bool ret; + + spin_lock_irq(&dev->power.lock); + + ret = !pm_runtime_enabled(dev); + if (ret && dev->power.last_status == RPM_INVALID) + dev->power.last_status = RPM_BLOCKED; + + spin_unlock_irq(&dev->power.lock); + + return ret; +} + +void pm_runtime_unblock(struct device *dev) +{ + spin_lock_irq(&dev->power.lock); + + if (dev->power.last_status == RPM_BLOCKED) + dev->power.last_status = RPM_INVALID; + + spin_unlock_irq(&dev->power.lock); +} + void __pm_runtime_disable(struct device *dev, bool check_resume) { spin_lock_irq(&dev->power.lock); @@ -1152,8 +1523,8 @@ void __pm_runtime_disable(struct device *dev, bool check_resume) * means there probably is some I/O to process and disabling runtime PM * shouldn't prevent the device from processing the I/O. */ - if (check_resume && dev->power.request_pending - && dev->power.request == RPM_REQ_RESUME) { + if (check_resume && dev->power.request_pending && + dev->power.request == RPM_REQ_RESUME) { /* * Prevent suspends and idle notifications from being carried * out after we have woken up the device. @@ -1165,8 +1536,13 @@ void __pm_runtime_disable(struct device *dev, bool check_resume) pm_runtime_put_noidle(dev); } - if (!dev->power.disable_depth++) + /* Update time accounting before disabling PM-runtime. */ + update_pm_runtime_accounting(dev); + + if (!dev->power.disable_depth++) { __pm_runtime_barrier(dev); + dev->power.last_status = dev->power.runtime_status; + } out: spin_unlock_irq(&dev->power.lock); @@ -1183,22 +1559,107 @@ void pm_runtime_enable(struct device *dev) spin_lock_irqsave(&dev->power.lock, flags); - if (dev->power.disable_depth > 0) - dev->power.disable_depth--; - else + if (!dev->power.disable_depth) { dev_warn(dev, "Unbalanced %s!\n", __func__); + goto out; + } + + if (--dev->power.disable_depth > 0) + goto out; + + if (dev->power.last_status == RPM_BLOCKED) { + dev_warn(dev, "Attempt to enable runtime PM when it is blocked\n"); + dump_stack(); + } + dev->power.last_status = RPM_INVALID; + dev->power.accounting_timestamp = ktime_get_mono_fast_ns(); + if (dev->power.runtime_status == RPM_SUSPENDED && + !dev->power.ignore_children && + atomic_read(&dev->power.child_count) > 0) + dev_warn(dev, "Enabling runtime PM for inactive device with active children\n"); + +out: spin_unlock_irqrestore(&dev->power.lock, flags); } EXPORT_SYMBOL_GPL(pm_runtime_enable); +static void pm_runtime_set_suspended_action(void *data) +{ + pm_runtime_set_suspended(data); +} + +/** + * devm_pm_runtime_set_active_enabled - set_active version of devm_pm_runtime_enable. + * + * @dev: Device to handle. + */ +int devm_pm_runtime_set_active_enabled(struct device *dev) +{ + int err; + + err = pm_runtime_set_active(dev); + if (err) + return err; + + err = devm_add_action_or_reset(dev, pm_runtime_set_suspended_action, dev); + if (err) + return err; + + return devm_pm_runtime_enable(dev); +} +EXPORT_SYMBOL_GPL(devm_pm_runtime_set_active_enabled); + +static void pm_runtime_disable_action(void *data) +{ + pm_runtime_dont_use_autosuspend(data); + pm_runtime_disable(data); +} + +/** + * devm_pm_runtime_enable - devres-enabled version of pm_runtime_enable. + * + * NOTE: this will also handle calling pm_runtime_dont_use_autosuspend() for + * you at driver exit time if needed. + * + * @dev: Device to handle. + */ +int devm_pm_runtime_enable(struct device *dev) +{ + pm_runtime_enable(dev); + + return devm_add_action_or_reset(dev, pm_runtime_disable_action, dev); +} +EXPORT_SYMBOL_GPL(devm_pm_runtime_enable); + +static void pm_runtime_put_noidle_action(void *data) +{ + pm_runtime_put_noidle(data); +} + +/** + * devm_pm_runtime_get_noresume - devres-enabled version of pm_runtime_get_noresume. + * + * @dev: Device to handle. + */ +int devm_pm_runtime_get_noresume(struct device *dev) +{ + pm_runtime_get_noresume(dev); + + return devm_add_action_or_reset(dev, pm_runtime_put_noidle_action, dev); +} +EXPORT_SYMBOL_GPL(devm_pm_runtime_get_noresume); + /** * pm_runtime_forbid - Block runtime PM of a device. * @dev: Device to handle. * - * Increase the device's usage count and clear its power.runtime_auto flag, - * so that it cannot be suspended at run time until pm_runtime_allow() is called - * for it. + * Resume @dev if already suspended and block runtime suspend of @dev in such + * a way that it can be unblocked via the /sys/devices/.../power/control + * interface, or otherwise by calling pm_runtime_allow(). + * + * Calling this function many times in a row has the same effect as calling it + * once. */ void pm_runtime_forbid(struct device *dev) { @@ -1219,17 +1680,28 @@ EXPORT_SYMBOL_GPL(pm_runtime_forbid); * pm_runtime_allow - Unblock runtime PM of a device. * @dev: Device to handle. * - * Decrease the device's usage count and set its power.runtime_auto flag. + * Unblock runtime suspend of @dev after it has been blocked by + * pm_runtime_forbid() (for instance, if it has been blocked via the + * /sys/devices/.../power/control interface), check if @dev can be + * suspended and suspend it in that case. + * + * Calling this function many times in a row has the same effect as calling it + * once. */ void pm_runtime_allow(struct device *dev) { + int ret; + spin_lock_irq(&dev->power.lock); if (dev->power.runtime_auto) goto out; dev->power.runtime_auto = true; - if (atomic_dec_and_test(&dev->power.usage_count)) - rpm_idle(dev, RPM_AUTO); + ret = rpm_drop_usage_count(dev); + if (ret == 0) + rpm_idle(dev, RPM_AUTO | RPM_ASYNC); + else if (ret > 0) + trace_rpm_usage(dev, RPM_AUTO | RPM_ASYNC); out: spin_unlock_irq(&dev->power.lock); @@ -1269,6 +1741,7 @@ void pm_runtime_irq_safe(struct device *dev) { if (dev->parent) pm_runtime_get_sync(dev->parent); + spin_lock_irq(&dev->power.lock); dev->power.irq_safe = 1; spin_unlock_irq(&dev->power.lock); @@ -1297,6 +1770,8 @@ static void update_autosuspend(struct device *dev, int old_delay, int old_use) if (!old_use || old_delay >= 0) { atomic_inc(&dev->power.usage_count); rpm_resume(dev, 0); + } else { + trace_rpm_usage(dev, 0); } } @@ -1362,6 +1837,7 @@ EXPORT_SYMBOL_GPL(__pm_runtime_use_autosuspend); void pm_runtime_init(struct device *dev) { dev->power.runtime_status = RPM_SUSPENDED; + dev->power.last_status = RPM_INVALID; dev->power.idle_notification = false; dev->power.disable_depth = 1; @@ -1376,27 +1852,264 @@ void pm_runtime_init(struct device *dev) dev->power.request_pending = false; dev->power.request = RPM_REQ_NONE; dev->power.deferred_resume = false; - dev->power.accounting_timestamp = jiffies; + dev->power.needs_force_resume = false; INIT_WORK(&dev->power.work, pm_runtime_work); dev->power.timer_expires = 0; - setup_timer(&dev->power.suspend_timer, pm_suspend_timer_fn, - (unsigned long)dev); + hrtimer_setup(&dev->power.suspend_timer, pm_suspend_timer_fn, CLOCK_MONOTONIC, + HRTIMER_MODE_ABS); init_waitqueue_head(&dev->power.wait_queue); } /** + * pm_runtime_reinit - Re-initialize runtime PM fields in given device object. + * @dev: Device object to re-initialize. + */ +void pm_runtime_reinit(struct device *dev) +{ + if (!pm_runtime_enabled(dev)) { + if (dev->power.runtime_status == RPM_ACTIVE) + pm_runtime_set_suspended(dev); + if (dev->power.irq_safe) { + spin_lock_irq(&dev->power.lock); + dev->power.irq_safe = 0; + spin_unlock_irq(&dev->power.lock); + if (dev->parent) + pm_runtime_put(dev->parent); + } + } + /* + * Clear power.needs_force_resume in case it has been set by + * pm_runtime_force_suspend() invoked from a driver remove callback. + */ + dev->power.needs_force_resume = false; +} + +/** * pm_runtime_remove - Prepare for removing a device from device hierarchy. * @dev: Device object being removed from device hierarchy. */ void pm_runtime_remove(struct device *dev) { __pm_runtime_disable(dev, false); + pm_runtime_reinit(dev); +} - /* Change the status back to 'suspended' to match the initial status. */ - if (dev->power.runtime_status == RPM_ACTIVE) +/** + * pm_runtime_get_suppliers - Resume and reference-count supplier devices. + * @dev: Consumer device. + */ +void pm_runtime_get_suppliers(struct device *dev) +{ + struct device_link *link; + int idx; + + idx = device_links_read_lock(); + + dev_for_each_link_to_supplier(link, dev) + if (device_link_test(link, DL_FLAG_PM_RUNTIME)) { + link->supplier_preactivated = true; + pm_runtime_get_sync(link->supplier); + } + + device_links_read_unlock(idx); +} + +/** + * pm_runtime_put_suppliers - Drop references to supplier devices. + * @dev: Consumer device. + */ +void pm_runtime_put_suppliers(struct device *dev) +{ + struct device_link *link; + int idx; + + idx = device_links_read_lock(); + + list_for_each_entry_rcu(link, &dev->links.suppliers, c_node, + device_links_read_lock_held()) + if (link->supplier_preactivated) { + link->supplier_preactivated = false; + pm_runtime_put(link->supplier); + } + + device_links_read_unlock(idx); +} + +void pm_runtime_new_link(struct device *dev) +{ + spin_lock_irq(&dev->power.lock); + dev->power.links_count++; + spin_unlock_irq(&dev->power.lock); +} + +static void pm_runtime_drop_link_count(struct device *dev) +{ + spin_lock_irq(&dev->power.lock); + WARN_ON(dev->power.links_count == 0); + dev->power.links_count--; + spin_unlock_irq(&dev->power.lock); +} + +/** + * pm_runtime_drop_link - Prepare for device link removal. + * @link: Device link going away. + * + * Drop the link count of the consumer end of @link and decrement the supplier + * device's runtime PM usage counter as many times as needed to drop all of the + * PM runtime reference to it from the consumer. + */ +void pm_runtime_drop_link(struct device_link *link) +{ + if (!device_link_test(link, DL_FLAG_PM_RUNTIME)) + return; + + pm_runtime_drop_link_count(link->consumer); + pm_runtime_release_supplier(link); + pm_request_idle(link->supplier); +} + +static pm_callback_t get_callback(struct device *dev, size_t cb_offset) +{ + /* + * Setting power.strict_midlayer means that the middle layer + * code does not want its runtime PM callbacks to be invoked via + * pm_runtime_force_suspend() and pm_runtime_force_resume(), so + * return a direct pointer to the driver callback in that case. + */ + if (dev_pm_strict_midlayer_is_set(dev)) + return __rpm_get_driver_callback(dev, cb_offset); + + return __rpm_get_callback(dev, cb_offset); +} + +#define GET_CALLBACK(dev, callback) \ + get_callback(dev, offsetof(struct dev_pm_ops, callback)) + +/** + * pm_runtime_force_suspend - Force a device into suspend state if needed. + * @dev: Device to suspend. + * + * Disable runtime PM so we safely can check the device's runtime PM status and + * if it is active, invoke its ->runtime_suspend callback to suspend it and + * change its runtime PM status field to RPM_SUSPENDED. Also, if the device's + * usage and children counters don't indicate that the device was in use before + * the system-wide transition under way, decrement its parent's children counter + * (if there is a parent). Keep runtime PM disabled to preserve the state + * unless we encounter errors. + * + * Typically this function may be invoked from a system suspend callback to make + * sure the device is put into low power state and it should only be used during + * system-wide PM transitions to sleep states. It assumes that the analogous + * pm_runtime_force_resume() will be used to resume the device. + */ +int pm_runtime_force_suspend(struct device *dev) +{ + int (*callback)(struct device *); + int ret; + + pm_runtime_disable(dev); + if (pm_runtime_status_suspended(dev) || dev->power.needs_force_resume) + return 0; + + callback = GET_CALLBACK(dev, runtime_suspend); + + dev_pm_enable_wake_irq_check(dev, true); + ret = callback ? callback(dev) : 0; + if (ret) + goto err; + + dev_pm_enable_wake_irq_complete(dev); + + /* + * If the device can stay in suspend after the system-wide transition + * to the working state that will follow, drop the children counter of + * its parent and the usage counters of its suppliers. Otherwise, set + * power.needs_force_resume to let pm_runtime_force_resume() know that + * the device needs to be taken care of and to prevent this function + * from handling the device again in case the device is passed to it + * once more subsequently. + */ + if (pm_runtime_need_not_resume(dev)) pm_runtime_set_suspended(dev); - if (dev->power.irq_safe && dev->parent) - pm_runtime_put(dev->parent); + else + dev->power.needs_force_resume = true; + + return 0; + +err: + dev_pm_disable_wake_irq_check(dev, true); + pm_runtime_enable(dev); + return ret; +} +EXPORT_SYMBOL_GPL(pm_runtime_force_suspend); + +#ifdef CONFIG_PM_SLEEP + +/** + * pm_runtime_force_resume - Force a device into resume state if needed. + * @dev: Device to resume. + * + * This function expects that either pm_runtime_force_suspend() has put the + * device into a low-power state prior to calling it, or the device had been + * runtime-suspended before the preceding system-wide suspend transition and it + * was left in suspend during that transition. + * + * The actions carried out by pm_runtime_force_suspend(), or by a runtime + * suspend in general, are reversed and the device is brought back into full + * power if it is expected to be used on system resume, which is the case when + * its needs_force_resume flag is set or when its smart_suspend flag is set and + * its runtime PM status is "active". + * + * In other cases, the resume is deferred to be managed via runtime PM. + * + * Typically, this function may be invoked from a system resume callback. + */ +int pm_runtime_force_resume(struct device *dev) +{ + int (*callback)(struct device *); + int ret = 0; + + if (!dev->power.needs_force_resume && (!dev_pm_smart_suspend(dev) || + pm_runtime_status_suspended(dev))) + goto out; + + callback = GET_CALLBACK(dev, runtime_resume); + + dev_pm_disable_wake_irq_check(dev, false); + ret = callback ? callback(dev) : 0; + if (ret) { + pm_runtime_set_suspended(dev); + dev_pm_enable_wake_irq_check(dev, false); + goto out; + } + + pm_runtime_mark_last_busy(dev); + +out: + /* + * The smart_suspend flag can be cleared here because it is not going + * to be necessary until the next system-wide suspend transition that + * will update it again. + */ + dev->power.smart_suspend = false; + /* + * Also clear needs_force_resume to make this function skip devices that + * have been seen by it once. + */ + dev->power.needs_force_resume = false; + + pm_runtime_enable(dev); + return ret; } +EXPORT_SYMBOL_GPL(pm_runtime_force_resume); + +bool pm_runtime_need_not_resume(struct device *dev) +{ + return atomic_read(&dev->power.usage_count) <= 1 && + (atomic_read(&dev->power.child_count) == 0 || + dev->power.ignore_children); +} + +#endif /* CONFIG_PM_SLEEP */ diff --git a/drivers/base/power/sysfs.c b/drivers/base/power/sysfs.c index a53ebd265701..13b31a3adc77 100644 --- a/drivers/base/power/sysfs.c +++ b/drivers/base/power/sysfs.c @@ -1,8 +1,7 @@ -/* - * drivers/base/power/sysfs.c - sysfs entries for device PM - */ - +// SPDX-License-Identifier: GPL-2.0 +/* sysfs entries for device PM */ #include <linux/device.h> +#include <linux/kobject.h> #include <linux/string.h> #include <linux/export.h> #include <linux/pm_qos.h> @@ -92,36 +91,26 @@ * wakeup_count - Report the number of wakeup events related to the device */ -static const char enabled[] = "enabled"; -static const char disabled[] = "disabled"; - const char power_group_name[] = "power"; EXPORT_SYMBOL_GPL(power_group_name); -#ifdef CONFIG_PM_RUNTIME static const char ctrl_auto[] = "auto"; static const char ctrl_on[] = "on"; static ssize_t control_show(struct device *dev, struct device_attribute *attr, char *buf) { - return sprintf(buf, "%s\n", - dev->power.runtime_auto ? ctrl_auto : ctrl_on); + return sysfs_emit(buf, "%s\n", + dev->power.runtime_auto ? ctrl_auto : ctrl_on); } static ssize_t control_store(struct device * dev, struct device_attribute *attr, const char * buf, size_t n) { - char *cp; - int len = n; - - cp = memchr(buf, '\n', n); - if (cp) - len = cp - buf; device_lock(dev); - if (len == sizeof ctrl_auto - 1 && strncmp(buf, ctrl_auto, len) == 0) + if (sysfs_streq(buf, ctrl_auto)) pm_runtime_allow(dev); - else if (len == sizeof ctrl_on - 1 && strncmp(buf, ctrl_on, len) == 0) + else if (sysfs_streq(buf, ctrl_on)) pm_runtime_forbid(dev); else n = -EINVAL; @@ -129,73 +118,74 @@ static ssize_t control_store(struct device * dev, struct device_attribute *attr, return n; } -static DEVICE_ATTR(control, 0644, control_show, control_store); +static DEVICE_ATTR_RW(control); -static ssize_t rtpm_active_time_show(struct device *dev, - struct device_attribute *attr, char *buf) +static ssize_t runtime_active_time_show(struct device *dev, + struct device_attribute *attr, + char *buf) { - int ret; - spin_lock_irq(&dev->power.lock); - update_pm_runtime_accounting(dev); - ret = sprintf(buf, "%i\n", jiffies_to_msecs(dev->power.active_jiffies)); - spin_unlock_irq(&dev->power.lock); - return ret; + u64 tmp = pm_runtime_active_time(dev); + + do_div(tmp, NSEC_PER_MSEC); + + return sysfs_emit(buf, "%llu\n", tmp); } -static DEVICE_ATTR(runtime_active_time, 0444, rtpm_active_time_show, NULL); +static DEVICE_ATTR_RO(runtime_active_time); -static ssize_t rtpm_suspended_time_show(struct device *dev, - struct device_attribute *attr, char *buf) +static ssize_t runtime_suspended_time_show(struct device *dev, + struct device_attribute *attr, + char *buf) { - int ret; - spin_lock_irq(&dev->power.lock); - update_pm_runtime_accounting(dev); - ret = sprintf(buf, "%i\n", - jiffies_to_msecs(dev->power.suspended_jiffies)); - spin_unlock_irq(&dev->power.lock); - return ret; + u64 tmp = pm_runtime_suspended_time(dev); + + do_div(tmp, NSEC_PER_MSEC); + + return sysfs_emit(buf, "%llu\n", tmp); } -static DEVICE_ATTR(runtime_suspended_time, 0444, rtpm_suspended_time_show, NULL); +static DEVICE_ATTR_RO(runtime_suspended_time); -static ssize_t rtpm_status_show(struct device *dev, - struct device_attribute *attr, char *buf) +static ssize_t runtime_status_show(struct device *dev, + struct device_attribute *attr, char *buf) { - const char *p; + const char *output; if (dev->power.runtime_error) { - p = "error\n"; + output = "error"; } else if (dev->power.disable_depth) { - p = "unsupported\n"; + output = "unsupported"; } else { switch (dev->power.runtime_status) { case RPM_SUSPENDED: - p = "suspended\n"; + output = "suspended"; break; case RPM_SUSPENDING: - p = "suspending\n"; + output = "suspending"; break; case RPM_RESUMING: - p = "resuming\n"; + output = "resuming"; break; case RPM_ACTIVE: - p = "active\n"; + output = "active"; break; default: return -EIO; } } - return sprintf(buf, p); + return sysfs_emit(buf, "%s\n", output); } -static DEVICE_ATTR(runtime_status, 0444, rtpm_status_show, NULL); +static DEVICE_ATTR_RO(runtime_status); static ssize_t autosuspend_delay_ms_show(struct device *dev, - struct device_attribute *attr, char *buf) + struct device_attribute *attr, + char *buf) { if (!dev->power.use_autosuspend) return -EIO; - return sprintf(buf, "%d\n", dev->power.autosuspend_delay); + + return sysfs_emit(buf, "%d\n", dev->power.autosuspend_delay); } static ssize_t autosuspend_delay_ms_store(struct device *dev, @@ -206,7 +196,7 @@ static ssize_t autosuspend_delay_ms_store(struct device *dev, if (!dev->power.use_autosuspend) return -EIO; - if (strict_strtol(buf, 10, &delay) != 0 || delay != (int) delay) + if (kstrtol(buf, 10, &delay) != 0 || delay != (int) delay) return -EINVAL; device_lock(dev); @@ -215,73 +205,102 @@ static ssize_t autosuspend_delay_ms_store(struct device *dev, return n; } -static DEVICE_ATTR(autosuspend_delay_ms, 0644, autosuspend_delay_ms_show, - autosuspend_delay_ms_store); +static DEVICE_ATTR_RW(autosuspend_delay_ms); -static ssize_t pm_qos_latency_show(struct device *dev, - struct device_attribute *attr, char *buf) +static ssize_t pm_qos_resume_latency_us_show(struct device *dev, + struct device_attribute *attr, + char *buf) { - return sprintf(buf, "%d\n", dev_pm_qos_requested_latency(dev)); + s32 value = dev_pm_qos_requested_resume_latency(dev); + + if (value == 0) + return sysfs_emit(buf, "n/a\n"); + if (value == PM_QOS_RESUME_LATENCY_NO_CONSTRAINT) + value = 0; + + return sysfs_emit(buf, "%d\n", value); } -static ssize_t pm_qos_latency_store(struct device *dev, - struct device_attribute *attr, - const char *buf, size_t n) +static ssize_t pm_qos_resume_latency_us_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t n) { s32 value; int ret; - if (kstrtos32(buf, 0, &value)) - return -EINVAL; - - if (value < 0) + if (!kstrtos32(buf, 0, &value)) { + /* + * Prevent users from writing negative or "no constraint" values + * directly. + */ + if (value < 0 || value == PM_QOS_RESUME_LATENCY_NO_CONSTRAINT) + return -EINVAL; + + if (value == 0) + value = PM_QOS_RESUME_LATENCY_NO_CONSTRAINT; + } else if (sysfs_streq(buf, "n/a")) { + value = 0; + } else { return -EINVAL; + } - ret = dev_pm_qos_update_request(dev->power.qos->latency_req, value); + ret = dev_pm_qos_update_request(dev->power.qos->resume_latency_req, + value); return ret < 0 ? ret : n; } -static DEVICE_ATTR(pm_qos_resume_latency_us, 0644, - pm_qos_latency_show, pm_qos_latency_store); +static DEVICE_ATTR_RW(pm_qos_resume_latency_us); -static ssize_t pm_qos_no_power_off_show(struct device *dev, - struct device_attribute *attr, - char *buf) +static ssize_t pm_qos_latency_tolerance_us_show(struct device *dev, + struct device_attribute *attr, + char *buf) { - return sprintf(buf, "%d\n", !!(dev_pm_qos_requested_flags(dev) - & PM_QOS_FLAG_NO_POWER_OFF)); + s32 value = dev_pm_qos_get_user_latency_tolerance(dev); + + if (value < 0) + return sysfs_emit(buf, "%s\n", "auto"); + if (value == PM_QOS_LATENCY_ANY) + return sysfs_emit(buf, "%s\n", "any"); + + return sysfs_emit(buf, "%d\n", value); } -static ssize_t pm_qos_no_power_off_store(struct device *dev, - struct device_attribute *attr, - const char *buf, size_t n) +static ssize_t pm_qos_latency_tolerance_us_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t n) { + s32 value; int ret; - if (kstrtoint(buf, 0, &ret)) - return -EINVAL; - - if (ret != 0 && ret != 1) - return -EINVAL; - - ret = dev_pm_qos_update_flags(dev, PM_QOS_FLAG_NO_POWER_OFF, ret); + if (kstrtos32(buf, 0, &value) == 0) { + /* Users can't write negative values directly */ + if (value < 0) + return -EINVAL; + } else { + if (sysfs_streq(buf, "auto")) + value = PM_QOS_LATENCY_TOLERANCE_NO_CONSTRAINT; + else if (sysfs_streq(buf, "any")) + value = PM_QOS_LATENCY_ANY; + else + return -EINVAL; + } + ret = dev_pm_qos_update_user_latency_tolerance(dev, value); return ret < 0 ? ret : n; } -static DEVICE_ATTR(pm_qos_no_power_off, 0644, - pm_qos_no_power_off_show, pm_qos_no_power_off_store); +static DEVICE_ATTR_RW(pm_qos_latency_tolerance_us); -static ssize_t pm_qos_remote_wakeup_show(struct device *dev, - struct device_attribute *attr, - char *buf) +static ssize_t pm_qos_no_power_off_show(struct device *dev, + struct device_attribute *attr, + char *buf) { - return sprintf(buf, "%d\n", !!(dev_pm_qos_requested_flags(dev) - & PM_QOS_FLAG_REMOTE_WAKEUP)); + return sysfs_emit(buf, "%d\n", !!(dev_pm_qos_requested_flags(dev) + & PM_QOS_FLAG_NO_POWER_OFF)); } -static ssize_t pm_qos_remote_wakeup_store(struct device *dev, - struct device_attribute *attr, - const char *buf, size_t n) +static ssize_t pm_qos_no_power_off_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t n) { int ret; @@ -291,70 +310,66 @@ static ssize_t pm_qos_remote_wakeup_store(struct device *dev, if (ret != 0 && ret != 1) return -EINVAL; - ret = dev_pm_qos_update_flags(dev, PM_QOS_FLAG_REMOTE_WAKEUP, ret); + ret = dev_pm_qos_update_flags(dev, PM_QOS_FLAG_NO_POWER_OFF, ret); return ret < 0 ? ret : n; } -static DEVICE_ATTR(pm_qos_remote_wakeup, 0644, - pm_qos_remote_wakeup_show, pm_qos_remote_wakeup_store); -#endif /* CONFIG_PM_RUNTIME */ +static DEVICE_ATTR_RW(pm_qos_no_power_off); #ifdef CONFIG_PM_SLEEP -static ssize_t -wake_show(struct device * dev, struct device_attribute *attr, char * buf) +static const char _enabled[] = "enabled"; +static const char _disabled[] = "disabled"; + +static ssize_t wakeup_show(struct device *dev, struct device_attribute *attr, + char *buf) { - return sprintf(buf, "%s\n", device_can_wakeup(dev) - ? (device_may_wakeup(dev) ? enabled : disabled) - : ""); + return sysfs_emit(buf, "%s\n", device_can_wakeup(dev) + ? (device_may_wakeup(dev) ? _enabled : _disabled) + : ""); } -static ssize_t -wake_store(struct device * dev, struct device_attribute *attr, - const char * buf, size_t n) +static ssize_t wakeup_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t n) { - char *cp; - int len = n; - if (!device_can_wakeup(dev)) return -EINVAL; - cp = memchr(buf, '\n', n); - if (cp) - len = cp - buf; - if (len == sizeof enabled - 1 - && strncmp(buf, enabled, sizeof enabled - 1) == 0) + if (sysfs_streq(buf, _enabled)) device_set_wakeup_enable(dev, 1); - else if (len == sizeof disabled - 1 - && strncmp(buf, disabled, sizeof disabled - 1) == 0) + else if (sysfs_streq(buf, _disabled)) device_set_wakeup_enable(dev, 0); else return -EINVAL; return n; } -static DEVICE_ATTR(wakeup, 0644, wake_show, wake_store); +static DEVICE_ATTR_RW(wakeup); static ssize_t wakeup_count_show(struct device *dev, - struct device_attribute *attr, char *buf) + struct device_attribute *attr, char *buf) { - unsigned long count = 0; + unsigned long count; bool enabled = false; spin_lock_irq(&dev->power.lock); if (dev->power.wakeup) { - count = dev->power.wakeup->event_count; + count = dev->power.wakeup->wakeup_count; enabled = true; } spin_unlock_irq(&dev->power.lock); - return enabled ? sprintf(buf, "%lu\n", count) : sprintf(buf, "\n"); + + if (!enabled) + return sysfs_emit(buf, "\n"); + return sysfs_emit(buf, "%lu\n", count); } -static DEVICE_ATTR(wakeup_count, 0444, wakeup_count_show, NULL); +static DEVICE_ATTR_RO(wakeup_count); static ssize_t wakeup_active_count_show(struct device *dev, - struct device_attribute *attr, char *buf) + struct device_attribute *attr, + char *buf) { - unsigned long count = 0; + unsigned long count; bool enabled = false; spin_lock_irq(&dev->power.lock); @@ -363,16 +378,19 @@ static ssize_t wakeup_active_count_show(struct device *dev, enabled = true; } spin_unlock_irq(&dev->power.lock); - return enabled ? sprintf(buf, "%lu\n", count) : sprintf(buf, "\n"); + + if (!enabled) + return sysfs_emit(buf, "\n"); + return sysfs_emit(buf, "%lu\n", count); } -static DEVICE_ATTR(wakeup_active_count, 0444, wakeup_active_count_show, NULL); +static DEVICE_ATTR_RO(wakeup_active_count); static ssize_t wakeup_abort_count_show(struct device *dev, - struct device_attribute *attr, - char *buf) + struct device_attribute *attr, + char *buf) { - unsigned long count = 0; + unsigned long count; bool enabled = false; spin_lock_irq(&dev->power.lock); @@ -381,16 +399,19 @@ static ssize_t wakeup_abort_count_show(struct device *dev, enabled = true; } spin_unlock_irq(&dev->power.lock); - return enabled ? sprintf(buf, "%lu\n", count) : sprintf(buf, "\n"); + + if (!enabled) + return sysfs_emit(buf, "\n"); + return sysfs_emit(buf, "%lu\n", count); } -static DEVICE_ATTR(wakeup_abort_count, 0444, wakeup_abort_count_show, NULL); +static DEVICE_ATTR_RO(wakeup_abort_count); static ssize_t wakeup_expire_count_show(struct device *dev, struct device_attribute *attr, char *buf) { - unsigned long count = 0; + unsigned long count; bool enabled = false; spin_lock_irq(&dev->power.lock); @@ -399,15 +420,18 @@ static ssize_t wakeup_expire_count_show(struct device *dev, enabled = true; } spin_unlock_irq(&dev->power.lock); - return enabled ? sprintf(buf, "%lu\n", count) : sprintf(buf, "\n"); + + if (!enabled) + return sysfs_emit(buf, "\n"); + return sysfs_emit(buf, "%lu\n", count); } -static DEVICE_ATTR(wakeup_expire_count, 0444, wakeup_expire_count_show, NULL); +static DEVICE_ATTR_RO(wakeup_expire_count); static ssize_t wakeup_active_show(struct device *dev, - struct device_attribute *attr, char *buf) + struct device_attribute *attr, char *buf) { - unsigned int active = 0; + unsigned int active; bool enabled = false; spin_lock_irq(&dev->power.lock); @@ -416,15 +440,19 @@ static ssize_t wakeup_active_show(struct device *dev, enabled = true; } spin_unlock_irq(&dev->power.lock); - return enabled ? sprintf(buf, "%u\n", active) : sprintf(buf, "\n"); + + if (!enabled) + return sysfs_emit(buf, "\n"); + return sysfs_emit(buf, "%u\n", active); } -static DEVICE_ATTR(wakeup_active, 0444, wakeup_active_show, NULL); +static DEVICE_ATTR_RO(wakeup_active); -static ssize_t wakeup_total_time_show(struct device *dev, - struct device_attribute *attr, char *buf) +static ssize_t wakeup_total_time_ms_show(struct device *dev, + struct device_attribute *attr, + char *buf) { - s64 msec = 0; + s64 msec; bool enabled = false; spin_lock_irq(&dev->power.lock); @@ -433,15 +461,18 @@ static ssize_t wakeup_total_time_show(struct device *dev, enabled = true; } spin_unlock_irq(&dev->power.lock); - return enabled ? sprintf(buf, "%lld\n", msec) : sprintf(buf, "\n"); + + if (!enabled) + return sysfs_emit(buf, "\n"); + return sysfs_emit(buf, "%lld\n", msec); } -static DEVICE_ATTR(wakeup_total_time_ms, 0444, wakeup_total_time_show, NULL); +static DEVICE_ATTR_RO(wakeup_total_time_ms); -static ssize_t wakeup_max_time_show(struct device *dev, - struct device_attribute *attr, char *buf) +static ssize_t wakeup_max_time_ms_show(struct device *dev, + struct device_attribute *attr, char *buf) { - s64 msec = 0; + s64 msec; bool enabled = false; spin_lock_irq(&dev->power.lock); @@ -450,15 +481,19 @@ static ssize_t wakeup_max_time_show(struct device *dev, enabled = true; } spin_unlock_irq(&dev->power.lock); - return enabled ? sprintf(buf, "%lld\n", msec) : sprintf(buf, "\n"); + + if (!enabled) + return sysfs_emit(buf, "\n"); + return sysfs_emit(buf, "%lld\n", msec); } -static DEVICE_ATTR(wakeup_max_time_ms, 0444, wakeup_max_time_show, NULL); +static DEVICE_ATTR_RO(wakeup_max_time_ms); -static ssize_t wakeup_last_time_show(struct device *dev, - struct device_attribute *attr, char *buf) +static ssize_t wakeup_last_time_ms_show(struct device *dev, + struct device_attribute *attr, + char *buf) { - s64 msec = 0; + s64 msec; bool enabled = false; spin_lock_irq(&dev->power.lock); @@ -467,17 +502,20 @@ static ssize_t wakeup_last_time_show(struct device *dev, enabled = true; } spin_unlock_irq(&dev->power.lock); - return enabled ? sprintf(buf, "%lld\n", msec) : sprintf(buf, "\n"); + + if (!enabled) + return sysfs_emit(buf, "\n"); + return sysfs_emit(buf, "%lld\n", msec); } -static DEVICE_ATTR(wakeup_last_time_ms, 0444, wakeup_last_time_show, NULL); +static DEVICE_ATTR_RO(wakeup_last_time_ms); #ifdef CONFIG_PM_AUTOSLEEP -static ssize_t wakeup_prevent_sleep_time_show(struct device *dev, - struct device_attribute *attr, - char *buf) +static ssize_t wakeup_prevent_sleep_time_ms_show(struct device *dev, + struct device_attribute *attr, + char *buf) { - s64 msec = 0; + s64 msec; bool enabled = false; spin_lock_irq(&dev->power.lock); @@ -486,95 +524,99 @@ static ssize_t wakeup_prevent_sleep_time_show(struct device *dev, enabled = true; } spin_unlock_irq(&dev->power.lock); - return enabled ? sprintf(buf, "%lld\n", msec) : sprintf(buf, "\n"); + + if (!enabled) + return sysfs_emit(buf, "\n"); + return sysfs_emit(buf, "%lld\n", msec); } -static DEVICE_ATTR(wakeup_prevent_sleep_time_ms, 0444, - wakeup_prevent_sleep_time_show, NULL); +static DEVICE_ATTR_RO(wakeup_prevent_sleep_time_ms); #endif /* CONFIG_PM_AUTOSLEEP */ -#endif /* CONFIG_PM_SLEEP */ -#ifdef CONFIG_PM_ADVANCED_DEBUG -#ifdef CONFIG_PM_RUNTIME +static inline int dpm_sysfs_wakeup_change_owner(struct device *dev, kuid_t kuid, + kgid_t kgid) +{ + if (dev->power.wakeup && dev->power.wakeup->dev) + return device_change_owner(dev->power.wakeup->dev, kuid, kgid); + return 0; +} -static ssize_t rtpm_usagecount_show(struct device *dev, - struct device_attribute *attr, char *buf) +#else /* CONFIG_PM_SLEEP */ +static inline int dpm_sysfs_wakeup_change_owner(struct device *dev, kuid_t kuid, + kgid_t kgid) { - return sprintf(buf, "%d\n", atomic_read(&dev->power.usage_count)); + return 0; } +#endif -static ssize_t rtpm_children_show(struct device *dev, +#ifdef CONFIG_PM_ADVANCED_DEBUG +static ssize_t runtime_usage_show(struct device *dev, struct device_attribute *attr, char *buf) { - return sprintf(buf, "%d\n", dev->power.ignore_children ? - 0 : atomic_read(&dev->power.child_count)); + return sysfs_emit(buf, "%d\n", atomic_read(&dev->power.usage_count)); } +static DEVICE_ATTR_RO(runtime_usage); -static ssize_t rtpm_enabled_show(struct device *dev, - struct device_attribute *attr, char *buf) +static ssize_t runtime_active_kids_show(struct device *dev, + struct device_attribute *attr, + char *buf) { - if ((dev->power.disable_depth) && (dev->power.runtime_auto == false)) - return sprintf(buf, "disabled & forbidden\n"); - else if (dev->power.disable_depth) - return sprintf(buf, "disabled\n"); - else if (dev->power.runtime_auto == false) - return sprintf(buf, "forbidden\n"); - return sprintf(buf, "enabled\n"); + return sysfs_emit(buf, "%d\n", dev->power.ignore_children ? + 0 : atomic_read(&dev->power.child_count)); } +static DEVICE_ATTR_RO(runtime_active_kids); -static DEVICE_ATTR(runtime_usage, 0444, rtpm_usagecount_show, NULL); -static DEVICE_ATTR(runtime_active_kids, 0444, rtpm_children_show, NULL); -static DEVICE_ATTR(runtime_enabled, 0444, rtpm_enabled_show, NULL); +static ssize_t runtime_enabled_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + const char *output; -#endif + if (dev->power.disable_depth && !dev->power.runtime_auto) + output = "disabled & forbidden"; + else if (dev->power.disable_depth) + output = "disabled"; + else if (!dev->power.runtime_auto) + output = "forbidden"; + else + output = "enabled"; -#ifdef CONFIG_PM_SLEEP + return sysfs_emit(buf, "%s\n", output); +} +static DEVICE_ATTR_RO(runtime_enabled); +#ifdef CONFIG_PM_SLEEP static ssize_t async_show(struct device *dev, struct device_attribute *attr, char *buf) { - return sprintf(buf, "%s\n", - device_async_suspend_enabled(dev) ? enabled : disabled); + return sysfs_emit(buf, "%s\n", + device_async_suspend_enabled(dev) ? + _enabled : _disabled); } static ssize_t async_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t n) { - char *cp; - int len = n; - - cp = memchr(buf, '\n', n); - if (cp) - len = cp - buf; - if (len == sizeof enabled - 1 && strncmp(buf, enabled, len) == 0) + if (sysfs_streq(buf, _enabled)) device_enable_async_suspend(dev); - else if (len == sizeof disabled - 1 && strncmp(buf, disabled, len) == 0) + else if (sysfs_streq(buf, _disabled)) device_disable_async_suspend(dev); else return -EINVAL; return n; } -static DEVICE_ATTR(async, 0644, async_show, async_store); +static DEVICE_ATTR_RW(async); -#endif +#endif /* CONFIG_PM_SLEEP */ #endif /* CONFIG_PM_ADVANCED_DEBUG */ static struct attribute *power_attrs[] = { -#ifdef CONFIG_PM_ADVANCED_DEBUG -#ifdef CONFIG_PM_SLEEP +#if defined(CONFIG_PM_ADVANCED_DEBUG) && defined(CONFIG_PM_SLEEP) &dev_attr_async.attr, #endif -#ifdef CONFIG_PM_RUNTIME - &dev_attr_runtime_status.attr, - &dev_attr_runtime_usage.attr, - &dev_attr_runtime_active_kids.attr, - &dev_attr_runtime_enabled.attr, -#endif -#endif /* CONFIG_PM_ADVANCED_DEBUG */ NULL, }; -static struct attribute_group pm_attr_group = { +static const struct attribute_group pm_attr_group = { .name = power_group_name, .attrs = power_attrs, }; @@ -596,47 +638,52 @@ static struct attribute *wakeup_attrs[] = { #endif NULL, }; -static struct attribute_group pm_wakeup_attr_group = { +static const struct attribute_group pm_wakeup_attr_group = { .name = power_group_name, .attrs = wakeup_attrs, }; static struct attribute *runtime_attrs[] = { -#ifdef CONFIG_PM_RUNTIME -#ifndef CONFIG_PM_ADVANCED_DEBUG &dev_attr_runtime_status.attr, -#endif &dev_attr_control.attr, &dev_attr_runtime_suspended_time.attr, &dev_attr_runtime_active_time.attr, &dev_attr_autosuspend_delay_ms.attr, -#endif /* CONFIG_PM_RUNTIME */ +#ifdef CONFIG_PM_ADVANCED_DEBUG + &dev_attr_runtime_usage.attr, + &dev_attr_runtime_active_kids.attr, + &dev_attr_runtime_enabled.attr, +#endif NULL, }; -static struct attribute_group pm_runtime_attr_group = { +static const struct attribute_group pm_runtime_attr_group = { .name = power_group_name, .attrs = runtime_attrs, }; -static struct attribute *pm_qos_latency_attrs[] = { -#ifdef CONFIG_PM_RUNTIME +static struct attribute *pm_qos_resume_latency_attrs[] = { &dev_attr_pm_qos_resume_latency_us.attr, -#endif /* CONFIG_PM_RUNTIME */ NULL, }; -static struct attribute_group pm_qos_latency_attr_group = { +static const struct attribute_group pm_qos_resume_latency_attr_group = { .name = power_group_name, - .attrs = pm_qos_latency_attrs, + .attrs = pm_qos_resume_latency_attrs, +}; + +static struct attribute *pm_qos_latency_tolerance_attrs[] = { + &dev_attr_pm_qos_latency_tolerance_us.attr, + NULL, +}; +static const struct attribute_group pm_qos_latency_tolerance_attr_group = { + .name = power_group_name, + .attrs = pm_qos_latency_tolerance_attrs, }; static struct attribute *pm_qos_flags_attrs[] = { -#ifdef CONFIG_PM_RUNTIME &dev_attr_pm_qos_no_power_off.attr, - &dev_attr_pm_qos_remote_wakeup.attr, -#endif /* CONFIG_PM_RUNTIME */ NULL, }; -static struct attribute_group pm_qos_flags_attr_group = { +static const struct attribute_group pm_qos_flags_attr_group = { .name = power_group_name, .attrs = pm_qos_flags_attrs, }; @@ -645,50 +692,109 @@ int dpm_sysfs_add(struct device *dev) { int rc; + /* No need to create PM sysfs if explicitly disabled. */ + if (device_pm_not_required(dev)) + return 0; + rc = sysfs_create_group(&dev->kobj, &pm_attr_group); if (rc) return rc; - if (pm_runtime_callbacks_present(dev)) { + if (!pm_runtime_has_no_callbacks(dev)) { rc = sysfs_merge_group(&dev->kobj, &pm_runtime_attr_group); if (rc) goto err_out; } - if (device_can_wakeup(dev)) { rc = sysfs_merge_group(&dev->kobj, &pm_wakeup_attr_group); - if (rc) { - if (pm_runtime_callbacks_present(dev)) - sysfs_unmerge_group(&dev->kobj, - &pm_runtime_attr_group); - goto err_out; - } + if (rc) + goto err_runtime; } + if (dev->power.set_latency_tolerance) { + rc = sysfs_merge_group(&dev->kobj, + &pm_qos_latency_tolerance_attr_group); + if (rc) + goto err_wakeup; + } + rc = pm_wakeup_source_sysfs_add(dev); + if (rc) + goto err_latency; return 0; + err_latency: + sysfs_unmerge_group(&dev->kobj, &pm_qos_latency_tolerance_attr_group); + err_wakeup: + sysfs_unmerge_group(&dev->kobj, &pm_wakeup_attr_group); + err_runtime: + sysfs_unmerge_group(&dev->kobj, &pm_runtime_attr_group); err_out: sysfs_remove_group(&dev->kobj, &pm_attr_group); return rc; } +int dpm_sysfs_change_owner(struct device *dev, kuid_t kuid, kgid_t kgid) +{ + int rc; + + if (device_pm_not_required(dev)) + return 0; + + rc = sysfs_group_change_owner(&dev->kobj, &pm_attr_group, kuid, kgid); + if (rc) + return rc; + + if (!pm_runtime_has_no_callbacks(dev)) { + rc = sysfs_group_change_owner( + &dev->kobj, &pm_runtime_attr_group, kuid, kgid); + if (rc) + return rc; + } + + if (device_can_wakeup(dev)) { + rc = sysfs_group_change_owner(&dev->kobj, &pm_wakeup_attr_group, + kuid, kgid); + if (rc) + return rc; + + rc = dpm_sysfs_wakeup_change_owner(dev, kuid, kgid); + if (rc) + return rc; + } + + if (dev->power.set_latency_tolerance) { + rc = sysfs_group_change_owner( + &dev->kobj, &pm_qos_latency_tolerance_attr_group, kuid, + kgid); + if (rc) + return rc; + } + return 0; +} + int wakeup_sysfs_add(struct device *dev) { - return sysfs_merge_group(&dev->kobj, &pm_wakeup_attr_group); + int ret = sysfs_merge_group(&dev->kobj, &pm_wakeup_attr_group); + + if (!ret) + kobject_uevent(&dev->kobj, KOBJ_CHANGE); + + return ret; } void wakeup_sysfs_remove(struct device *dev) { sysfs_unmerge_group(&dev->kobj, &pm_wakeup_attr_group); + kobject_uevent(&dev->kobj, KOBJ_CHANGE); } -int pm_qos_sysfs_add_latency(struct device *dev) +int pm_qos_sysfs_add_resume_latency(struct device *dev) { - return sysfs_merge_group(&dev->kobj, &pm_qos_latency_attr_group); + return sysfs_merge_group(&dev->kobj, &pm_qos_resume_latency_attr_group); } -void pm_qos_sysfs_remove_latency(struct device *dev) +void pm_qos_sysfs_remove_resume_latency(struct device *dev) { - sysfs_unmerge_group(&dev->kobj, &pm_qos_latency_attr_group); + sysfs_unmerge_group(&dev->kobj, &pm_qos_resume_latency_attr_group); } int pm_qos_sysfs_add_flags(struct device *dev) @@ -701,6 +807,17 @@ void pm_qos_sysfs_remove_flags(struct device *dev) sysfs_unmerge_group(&dev->kobj, &pm_qos_flags_attr_group); } +int pm_qos_sysfs_add_latency_tolerance(struct device *dev) +{ + return sysfs_merge_group(&dev->kobj, + &pm_qos_latency_tolerance_attr_group); +} + +void pm_qos_sysfs_remove_latency_tolerance(struct device *dev) +{ + sysfs_unmerge_group(&dev->kobj, &pm_qos_latency_tolerance_attr_group); +} + void rpm_sysfs_remove(struct device *dev) { sysfs_unmerge_group(&dev->kobj, &pm_runtime_attr_group); @@ -708,6 +825,9 @@ void rpm_sysfs_remove(struct device *dev) void dpm_sysfs_remove(struct device *dev) { + if (device_pm_not_required(dev)) + return; + sysfs_unmerge_group(&dev->kobj, &pm_qos_latency_tolerance_attr_group); dev_pm_qos_constraints_destroy(dev); rpm_sysfs_remove(dev); sysfs_unmerge_group(&dev->kobj, &pm_wakeup_attr_group); diff --git a/drivers/base/power/trace.c b/drivers/base/power/trace.c index d94a1f5121cf..d8da7195bb00 100644 --- a/drivers/base/power/trace.c +++ b/drivers/base/power/trace.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * drivers/base/power/trace.c * @@ -6,12 +7,15 @@ * Trace facility for suspend/resume problems, when none of the * devices may be working. */ +#define pr_fmt(fmt) "PM: " fmt -#include <linux/resume-trace.h> +#include <linux/pm-trace.h> #include <linux/export.h> #include <linux/rtc.h> +#include <linux/suspend.h> +#include <linux/init.h> -#include <asm/rtc.h> +#include <linux/mc146818rtc.h> #include "power.h" @@ -74,6 +78,9 @@ #define DEVSEED (7919) +bool pm_trace_rtc_abused __read_mostly; +EXPORT_SYMBOL_GPL(pm_trace_rtc_abused); + static unsigned int dev_hash_value; static int set_magic_time(unsigned int user, unsigned int file, unsigned int device) @@ -103,7 +110,8 @@ static int set_magic_time(unsigned int user, unsigned int file, unsigned int dev n /= 24; time.tm_min = (n % 20) * 3; n /= 20; - set_rtc_time(&time); + mc146818_set_time(&time); + pm_trace_rtc_abused = true; return n ? -1 : 0; } @@ -112,10 +120,12 @@ static unsigned int read_magic_time(void) struct rtc_time time; unsigned int val; - get_rtc_time(&time); - pr_info("RTC time: %2d:%02d:%02d, date: %02d/%02d/%02d\n", - time.tm_hour, time.tm_min, time.tm_sec, - time.tm_mon + 1, time.tm_mday, time.tm_year % 100); + if (mc146818_get_time(&time, 1000) < 0) { + pr_err("Unable to read current time from RTC\n"); + return 0; + } + + pr_info("RTC time: %ptRt, date: %ptRd\n", &time, &time); val = time.tm_year; /* 100 years */ if (val > 100) val -= 100; @@ -154,26 +164,29 @@ EXPORT_SYMBOL(set_trace_device); * it's not any guarantee, but it's a high _likelihood_ that * the match is valid). */ -void generate_resume_trace(const void *tracedata, unsigned int user) +void generate_pm_trace(const void *tracedata, unsigned int user) { unsigned short lineno = *(unsigned short *)tracedata; const char *file = *(const char **)(tracedata + 2); unsigned int user_hash_value, file_hash_value; + if (!x86_platform.legacy.rtc) + return; + user_hash_value = user % USERHASH; file_hash_value = hash_string(lineno, file, FILEHASH); set_magic_time(user_hash_value, file_hash_value, dev_hash_value); } -EXPORT_SYMBOL(generate_resume_trace); +EXPORT_SYMBOL(generate_pm_trace); -extern char __tracedata_start, __tracedata_end; +extern char __tracedata_start[], __tracedata_end[]; static int show_file_hash(unsigned int value) { int match; char *tracedata; match = 0; - for (tracedata = &__tracedata_start ; tracedata < &__tracedata_end ; + for (tracedata = __tracedata_start ; tracedata < __tracedata_end ; tracedata += 2 + sizeof(unsigned long)) { unsigned short lineno = *(unsigned short *)tracedata; const char *file = *(const char **)(tracedata + 2); @@ -225,10 +238,8 @@ int show_trace_dev_match(char *buf, size_t size) unsigned int hash = hash_string(DEVSEED, dev_name(dev), DEVHASH); if (hash == value) { - int len = snprintf(buf, size, "%s\n", + int len = scnprintf(buf, size, "%s\n", dev_driver_string(dev)); - if (len > size) - len = size; buf += len; ret += len; size -= len; @@ -239,17 +250,45 @@ int show_trace_dev_match(char *buf, size_t size) return ret; } -static int early_resume_init(void) +static int +pm_trace_notify(struct notifier_block *nb, unsigned long mode, void *_unused) { + switch (mode) { + case PM_POST_HIBERNATION: + case PM_POST_SUSPEND: + if (pm_trace_rtc_abused) { + pm_trace_rtc_abused = false; + pr_warn("Possible incorrect RTC due to pm_trace, please use 'ntpdate' or 'rdate' to reset it.\n"); + } + break; + default: + break; + } + return 0; +} + +static struct notifier_block pm_trace_nb = { + .notifier_call = pm_trace_notify, +}; + +static int __init early_resume_init(void) +{ + if (!x86_platform.legacy.rtc) + return 0; + hash_value_early_read = read_magic_time(); + register_pm_notifier(&pm_trace_nb); return 0; } -static int late_resume_init(void) +static int __init late_resume_init(void) { unsigned int val = hash_value_early_read; unsigned int user, file, dev; + if (!x86_platform.legacy.rtc) + return 0; + user = val % USERHASH; val = val / USERHASH; file = val % FILEHASH; diff --git a/drivers/base/power/wakeirq.c b/drivers/base/power/wakeirq.c new file mode 100644 index 000000000000..8aa28c08b289 --- /dev/null +++ b/drivers/base/power/wakeirq.c @@ -0,0 +1,388 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Device wakeirq helper functions */ +#include <linux/device.h> +#include <linux/interrupt.h> +#include <linux/irq.h> +#include <linux/slab.h> +#include <linux/pm_runtime.h> +#include <linux/pm_wakeirq.h> + +#include "power.h" + +/** + * dev_pm_attach_wake_irq - Attach device interrupt as a wake IRQ + * @dev: Device entry + * @wirq: Wake irq specific data + * + * Internal function to attach a dedicated wake-up interrupt as a wake IRQ. + */ +static int dev_pm_attach_wake_irq(struct device *dev, struct wake_irq *wirq) +{ + unsigned long flags; + + if (!dev || !wirq) + return -EINVAL; + + spin_lock_irqsave(&dev->power.lock, flags); + if (dev_WARN_ONCE(dev, dev->power.wakeirq, + "wake irq already initialized\n")) { + spin_unlock_irqrestore(&dev->power.lock, flags); + return -EEXIST; + } + + dev->power.wakeirq = wirq; + device_wakeup_attach_irq(dev, wirq); + + spin_unlock_irqrestore(&dev->power.lock, flags); + return 0; +} + +/** + * dev_pm_set_wake_irq - Attach device IO interrupt as wake IRQ + * @dev: Device entry + * @irq: Device IO interrupt + * + * Attach a device IO interrupt as a wake IRQ. The wake IRQ gets + * automatically configured for wake-up from suspend based + * on the device specific sysfs wakeup entry. Typically called + * during driver probe after calling device_init_wakeup(). + */ +int dev_pm_set_wake_irq(struct device *dev, int irq) +{ + struct wake_irq *wirq; + int err; + + if (irq < 0) + return -EINVAL; + + wirq = kzalloc(sizeof(*wirq), GFP_KERNEL); + if (!wirq) + return -ENOMEM; + + wirq->dev = dev; + wirq->irq = irq; + + err = dev_pm_attach_wake_irq(dev, wirq); + if (err) + kfree(wirq); + + return err; +} +EXPORT_SYMBOL_GPL(dev_pm_set_wake_irq); + +/** + * dev_pm_clear_wake_irq - Detach a device IO interrupt wake IRQ + * @dev: Device entry + * + * Detach a device wake IRQ and free resources. + * + * Note that it's OK for drivers to call this without calling + * dev_pm_set_wake_irq() as all the driver instances may not have + * a wake IRQ configured. This avoid adding wake IRQ specific + * checks into the drivers. + */ +void dev_pm_clear_wake_irq(struct device *dev) +{ + struct wake_irq *wirq = dev->power.wakeirq; + unsigned long flags; + + if (!wirq) + return; + + spin_lock_irqsave(&dev->power.lock, flags); + device_wakeup_detach_irq(dev); + dev->power.wakeirq = NULL; + spin_unlock_irqrestore(&dev->power.lock, flags); + + if (wirq->status & WAKE_IRQ_DEDICATED_ALLOCATED) { + free_irq(wirq->irq, wirq); + wirq->status &= ~WAKE_IRQ_DEDICATED_MASK; + } + kfree(wirq->name); + kfree(wirq); +} +EXPORT_SYMBOL_GPL(dev_pm_clear_wake_irq); + +static void devm_pm_clear_wake_irq(void *dev) +{ + dev_pm_clear_wake_irq(dev); +} + +/** + * devm_pm_set_wake_irq - device-managed variant of dev_pm_set_wake_irq + * @dev: Device entry + * @irq: Device IO interrupt + * + * + * Attach a device IO interrupt as a wake IRQ, same with dev_pm_set_wake_irq, + * but the device will be auto clear wake capability on driver detach. + */ +int devm_pm_set_wake_irq(struct device *dev, int irq) +{ + int ret; + + ret = dev_pm_set_wake_irq(dev, irq); + if (ret) + return ret; + + return devm_add_action_or_reset(dev, devm_pm_clear_wake_irq, dev); +} +EXPORT_SYMBOL_GPL(devm_pm_set_wake_irq); + +/** + * handle_threaded_wake_irq - Handler for dedicated wake-up interrupts + * @irq: Device specific dedicated wake-up interrupt + * @_wirq: Wake IRQ data + * + * Some devices have a separate wake-up interrupt in addition to the + * device IO interrupt. The wake-up interrupt signals that a device + * should be woken up from it's idle state. This handler uses device + * specific pm_runtime functions to wake the device, and then it's + * up to the device to do whatever it needs to. Note that as the + * device may need to restore context and start up regulators, we + * use a threaded IRQ. + * + * Also note that we are not resending the lost device interrupts. + * We assume that the wake-up interrupt just needs to wake-up the + * device, and then device's pm_runtime_resume() can deal with the + * situation. + */ +static irqreturn_t handle_threaded_wake_irq(int irq, void *_wirq) +{ + struct wake_irq *wirq = _wirq; + int res; + + /* Maybe abort suspend? */ + if (irqd_is_wakeup_set(irq_get_irq_data(irq))) { + pm_wakeup_event(wirq->dev, 0); + + return IRQ_HANDLED; + } + + /* We don't want RPM_ASYNC or RPM_NOWAIT here */ + res = pm_runtime_resume(wirq->dev); + if (res < 0) + dev_warn(wirq->dev, + "wake IRQ with no resume: %i\n", res); + + return IRQ_HANDLED; +} + +static int __dev_pm_set_dedicated_wake_irq(struct device *dev, int irq, unsigned int flag) +{ + struct wake_irq *wirq; + int err; + + if (irq < 0) + return -EINVAL; + + wirq = kzalloc(sizeof(*wirq), GFP_KERNEL); + if (!wirq) + return -ENOMEM; + + wirq->name = kasprintf(GFP_KERNEL, "%s:wakeup", dev_name(dev)); + if (!wirq->name) { + err = -ENOMEM; + goto err_free; + } + + wirq->dev = dev; + wirq->irq = irq; + + /* Prevent deferred spurious wakeirqs with disable_irq_nosync() */ + irq_set_status_flags(irq, IRQ_DISABLE_UNLAZY); + + /* + * Consumer device may need to power up and restore state + * so we use a threaded irq. + */ + err = request_threaded_irq(irq, NULL, handle_threaded_wake_irq, + IRQF_ONESHOT | IRQF_NO_AUTOEN, + wirq->name, wirq); + if (err) + goto err_free_name; + + err = dev_pm_attach_wake_irq(dev, wirq); + if (err) + goto err_free_irq; + + wirq->status = WAKE_IRQ_DEDICATED_ALLOCATED | flag; + + return err; + +err_free_irq: + free_irq(irq, wirq); +err_free_name: + kfree(wirq->name); +err_free: + kfree(wirq); + + return err; +} + +/** + * dev_pm_set_dedicated_wake_irq - Request a dedicated wake-up interrupt + * @dev: Device entry + * @irq: Device wake-up interrupt + * + * Unless your hardware has separate wake-up interrupts in addition + * to the device IO interrupts, you don't need this. + * + * Sets up a threaded interrupt handler for a device that has + * a dedicated wake-up interrupt in addition to the device IO + * interrupt. + */ +int dev_pm_set_dedicated_wake_irq(struct device *dev, int irq) +{ + return __dev_pm_set_dedicated_wake_irq(dev, irq, 0); +} +EXPORT_SYMBOL_GPL(dev_pm_set_dedicated_wake_irq); + +/** + * dev_pm_set_dedicated_wake_irq_reverse - Request a dedicated wake-up interrupt + * with reverse enable ordering + * @dev: Device entry + * @irq: Device wake-up interrupt + * + * Unless your hardware has separate wake-up interrupts in addition + * to the device IO interrupts, you don't need this. + * + * Sets up a threaded interrupt handler for a device that has a dedicated + * wake-up interrupt in addition to the device IO interrupt. It sets + * the status of WAKE_IRQ_DEDICATED_REVERSE to tell rpm_suspend() + * to enable dedicated wake-up interrupt after running the runtime suspend + * callback for @dev. + */ +int dev_pm_set_dedicated_wake_irq_reverse(struct device *dev, int irq) +{ + return __dev_pm_set_dedicated_wake_irq(dev, irq, WAKE_IRQ_DEDICATED_REVERSE); +} +EXPORT_SYMBOL_GPL(dev_pm_set_dedicated_wake_irq_reverse); + +/** + * dev_pm_enable_wake_irq_check - Checks and enables wake-up interrupt + * @dev: Device + * @can_change_status: Can change wake-up interrupt status + * + * Enables wakeirq conditionally. We need to enable wake-up interrupt + * lazily on the first rpm_suspend(). This is needed as the consumer device + * starts in RPM_SUSPENDED state, and the first pm_runtime_get() would + * otherwise try to disable already disabled wakeirq. The wake-up interrupt + * starts disabled with IRQ_NOAUTOEN set. + * + * Should be only called from rpm_suspend() and rpm_resume() path. + * Caller must hold &dev->power.lock to change wirq->status + */ +void dev_pm_enable_wake_irq_check(struct device *dev, + bool can_change_status) +{ + struct wake_irq *wirq = dev->power.wakeirq; + + if (!wirq || !(wirq->status & WAKE_IRQ_DEDICATED_MASK)) + return; + + if (likely(wirq->status & WAKE_IRQ_DEDICATED_MANAGED)) { + goto enable; + } else if (can_change_status) { + wirq->status |= WAKE_IRQ_DEDICATED_MANAGED; + goto enable; + } + + return; + +enable: + if (!can_change_status || !(wirq->status & WAKE_IRQ_DEDICATED_REVERSE)) { + enable_irq(wirq->irq); + wirq->status |= WAKE_IRQ_DEDICATED_ENABLED; + } +} + +/** + * dev_pm_disable_wake_irq_check - Checks and disables wake-up interrupt + * @dev: Device + * @cond_disable: if set, also check WAKE_IRQ_DEDICATED_REVERSE + * + * Disables wake-up interrupt conditionally based on status. + * Should be only called from rpm_suspend() and rpm_resume() path. + */ +void dev_pm_disable_wake_irq_check(struct device *dev, bool cond_disable) +{ + struct wake_irq *wirq = dev->power.wakeirq; + + if (!wirq || !(wirq->status & WAKE_IRQ_DEDICATED_MASK)) + return; + + if (cond_disable && (wirq->status & WAKE_IRQ_DEDICATED_REVERSE)) + return; + + if (wirq->status & WAKE_IRQ_DEDICATED_MANAGED) { + wirq->status &= ~WAKE_IRQ_DEDICATED_ENABLED; + disable_irq_nosync(wirq->irq); + } +} + +/** + * dev_pm_enable_wake_irq_complete - enable wake IRQ not enabled before + * @dev: Device using the wake IRQ + * + * Enable wake IRQ conditionally based on status, mainly used if want to + * enable wake IRQ after running ->runtime_suspend() which depends on + * WAKE_IRQ_DEDICATED_REVERSE. + * + * Should be only called from rpm_suspend() path. + */ +void dev_pm_enable_wake_irq_complete(struct device *dev) +{ + struct wake_irq *wirq = dev->power.wakeirq; + + if (!wirq || !(wirq->status & WAKE_IRQ_DEDICATED_MASK)) + return; + + if (wirq->status & WAKE_IRQ_DEDICATED_MANAGED && + wirq->status & WAKE_IRQ_DEDICATED_REVERSE) { + enable_irq(wirq->irq); + wirq->status |= WAKE_IRQ_DEDICATED_ENABLED; + } +} + +/** + * dev_pm_arm_wake_irq - Arm device wake-up + * @wirq: Device wake-up interrupt + * + * Sets up the wake-up event conditionally based on the + * device_may_wake(). + */ +void dev_pm_arm_wake_irq(struct wake_irq *wirq) +{ + if (!wirq) + return; + + if (device_may_wakeup(wirq->dev)) { + if (wirq->status & WAKE_IRQ_DEDICATED_ALLOCATED && + !(wirq->status & WAKE_IRQ_DEDICATED_ENABLED)) + enable_irq(wirq->irq); + + enable_irq_wake(wirq->irq); + } +} + +/** + * dev_pm_disarm_wake_irq - Disarm device wake-up + * @wirq: Device wake-up interrupt + * + * Clears up the wake-up event conditionally based on the + * device_may_wake(). + */ +void dev_pm_disarm_wake_irq(struct wake_irq *wirq) +{ + if (!wirq) + return; + + if (device_may_wakeup(wirq->dev)) { + disable_irq_wake(wirq->irq); + + if (wirq->status & WAKE_IRQ_DEDICATED_ALLOCATED && + !(wirq->status & WAKE_IRQ_DEDICATED_ENABLED)) + disable_irq_nosync(wirq->irq); + } +} diff --git a/drivers/base/power/wakeup.c b/drivers/base/power/wakeup.c index 2d56f4113ae7..1e1a0e7eeac5 100644 --- a/drivers/base/power/wakeup.c +++ b/drivers/base/power/wakeup.c @@ -1,29 +1,40 @@ +// SPDX-License-Identifier: GPL-2.0 /* * drivers/base/power/wakeup.c - System wakeup events framework * * Copyright (c) 2010 Rafael J. Wysocki <rjw@sisk.pl>, Novell Inc. - * - * This file is released under the GPLv2. */ +#define pr_fmt(fmt) "PM: " fmt #include <linux/device.h> #include <linux/slab.h> -#include <linux/sched.h> +#include <linux/sched/signal.h> #include <linux/capability.h> #include <linux/export.h> #include <linux/suspend.h> #include <linux/seq_file.h> #include <linux/debugfs.h> +#include <linux/pm_wakeirq.h> #include <trace/events/power.h> #include "power.h" +#define list_for_each_entry_rcu_locked(pos, head, member) \ + list_for_each_entry_rcu(pos, head, member, \ + srcu_read_lock_held(&wakeup_srcu)) /* * If set, the suspend/hibernate code will abort transitions to a sleep state * if wakeup events are registered during or immediately before the transition. */ bool events_check_enabled __read_mostly; +/* First wakeup IRQ seen by the kernel in the last cycle. */ +static unsigned int wakeup_irq[2] __read_mostly; +static DEFINE_RAW_SPINLOCK(wakeup_irq_lock); + +/* If greater than 0 and the system is suspending, terminate the suspend. */ +static atomic_t pm_abort_suspend __read_mostly; + /* * Combined counters of registered wakeup events and wakeup events in progress. * They need to be modified together atomically, so it's better to use one @@ -45,64 +56,91 @@ static void split_counters(unsigned int *cnt, unsigned int *inpr) /* A preserved old value of the events counter. */ static unsigned int saved_count; -static DEFINE_SPINLOCK(events_lock); +static DEFINE_RAW_SPINLOCK(events_lock); -static void pm_wakeup_timer_fn(unsigned long data); +static void pm_wakeup_timer_fn(struct timer_list *t); static LIST_HEAD(wakeup_sources); static DECLARE_WAIT_QUEUE_HEAD(wakeup_count_wait_queue); -/** - * wakeup_source_prepare - Prepare a new wakeup source for initialization. - * @ws: Wakeup source to prepare. - * @name: Pointer to the name of the new wakeup source. - * - * Callers must ensure that the @name string won't be freed when @ws is still in - * use. - */ -void wakeup_source_prepare(struct wakeup_source *ws, const char *name) -{ - if (ws) { - memset(ws, 0, sizeof(*ws)); - ws->name = name; - } -} -EXPORT_SYMBOL_GPL(wakeup_source_prepare); +DEFINE_STATIC_SRCU(wakeup_srcu); + +static struct wakeup_source deleted_ws = { + .name = "deleted", + .lock = __SPIN_LOCK_UNLOCKED(deleted_ws.lock), +}; + +static DEFINE_IDA(wakeup_ida); /** * wakeup_source_create - Create a struct wakeup_source object. * @name: Name of the new wakeup source. */ -struct wakeup_source *wakeup_source_create(const char *name) +static struct wakeup_source *wakeup_source_create(const char *name) { struct wakeup_source *ws; + const char *ws_name; + int id; - ws = kmalloc(sizeof(*ws), GFP_KERNEL); + ws = kzalloc(sizeof(*ws), GFP_KERNEL); if (!ws) - return NULL; + goto err_ws; + + ws_name = kstrdup_const(name, GFP_KERNEL); + if (!ws_name) + goto err_name; + ws->name = ws_name; + + id = ida_alloc(&wakeup_ida, GFP_KERNEL); + if (id < 0) + goto err_id; + ws->id = id; - wakeup_source_prepare(ws, name ? kstrdup(name, GFP_KERNEL) : NULL); return ws; + +err_id: + kfree_const(ws->name); +err_name: + kfree(ws); +err_ws: + return NULL; } -EXPORT_SYMBOL_GPL(wakeup_source_create); -/** - * wakeup_source_drop - Prepare a struct wakeup_source object for destruction. - * @ws: Wakeup source to prepare for destruction. - * - * Callers must ensure that __pm_stay_awake() or __pm_wakeup_event() will never - * be run in parallel with this function for the same wakeup source object. +/* + * Record wakeup_source statistics being deleted into a dummy wakeup_source. */ -void wakeup_source_drop(struct wakeup_source *ws) +static void wakeup_source_record(struct wakeup_source *ws) { - if (!ws) - return; + unsigned long flags; - del_timer_sync(&ws->timer); - __pm_relax(ws); + spin_lock_irqsave(&deleted_ws.lock, flags); + + if (ws->event_count) { + deleted_ws.total_time = + ktime_add(deleted_ws.total_time, ws->total_time); + deleted_ws.prevent_sleep_time = + ktime_add(deleted_ws.prevent_sleep_time, + ws->prevent_sleep_time); + deleted_ws.max_time = + ktime_compare(deleted_ws.max_time, ws->max_time) > 0 ? + deleted_ws.max_time : ws->max_time; + deleted_ws.event_count += ws->event_count; + deleted_ws.active_count += ws->active_count; + deleted_ws.relax_count += ws->relax_count; + deleted_ws.expire_count += ws->expire_count; + deleted_ws.wakeup_count += ws->wakeup_count; + } + + spin_unlock_irqrestore(&deleted_ws.lock, flags); +} + +static void wakeup_source_free(struct wakeup_source *ws) +{ + ida_free(&wakeup_ida, ws->id); + kfree_const(ws->name); + kfree(ws); } -EXPORT_SYMBOL_GPL(wakeup_source_drop); /** * wakeup_source_destroy - Destroy a struct wakeup_source object. @@ -110,22 +148,21 @@ EXPORT_SYMBOL_GPL(wakeup_source_drop); * * Use only for wakeup source objects created with wakeup_source_create(). */ -void wakeup_source_destroy(struct wakeup_source *ws) +static void wakeup_source_destroy(struct wakeup_source *ws) { if (!ws) return; - wakeup_source_drop(ws); - kfree(ws->name); - kfree(ws); + __pm_relax(ws); + wakeup_source_record(ws); + wakeup_source_free(ws); } -EXPORT_SYMBOL_GPL(wakeup_source_destroy); /** * wakeup_source_add - Add given object to the list of wakeup sources. * @ws: Wakeup source object to add to the list. */ -void wakeup_source_add(struct wakeup_source *ws) +static void wakeup_source_add(struct wakeup_source *ws) { unsigned long flags; @@ -133,46 +170,59 @@ void wakeup_source_add(struct wakeup_source *ws) return; spin_lock_init(&ws->lock); - setup_timer(&ws->timer, pm_wakeup_timer_fn, (unsigned long)ws); + timer_setup(&ws->timer, pm_wakeup_timer_fn, 0); ws->active = false; - ws->last_time = ktime_get(); - spin_lock_irqsave(&events_lock, flags); + raw_spin_lock_irqsave(&events_lock, flags); list_add_rcu(&ws->entry, &wakeup_sources); - spin_unlock_irqrestore(&events_lock, flags); + raw_spin_unlock_irqrestore(&events_lock, flags); } -EXPORT_SYMBOL_GPL(wakeup_source_add); /** * wakeup_source_remove - Remove given object from the wakeup sources list. * @ws: Wakeup source object to remove from the list. */ -void wakeup_source_remove(struct wakeup_source *ws) +static void wakeup_source_remove(struct wakeup_source *ws) { unsigned long flags; if (WARN_ON(!ws)) return; - spin_lock_irqsave(&events_lock, flags); + /* + * After shutting down the timer, wakeup_source_activate() will warn if + * the given wakeup source is passed to it. + */ + timer_shutdown_sync(&ws->timer); + + raw_spin_lock_irqsave(&events_lock, flags); list_del_rcu(&ws->entry); - spin_unlock_irqrestore(&events_lock, flags); - synchronize_rcu(); + raw_spin_unlock_irqrestore(&events_lock, flags); + synchronize_srcu(&wakeup_srcu); } -EXPORT_SYMBOL_GPL(wakeup_source_remove); /** * wakeup_source_register - Create wakeup source and add it to the list. + * @dev: Device this wakeup source is associated with (or NULL if virtual). * @name: Name of the wakeup source to register. */ -struct wakeup_source *wakeup_source_register(const char *name) +struct wakeup_source *wakeup_source_register(struct device *dev, + const char *name) { struct wakeup_source *ws; + int ret; ws = wakeup_source_create(name); - if (ws) + if (ws) { + if (!dev || device_is_registered(dev)) { + ret = wakeup_source_sysfs_add(dev, ws); + if (ret) { + wakeup_source_free(ws); + return NULL; + } + } wakeup_source_add(ws); - + } return ws; } EXPORT_SYMBOL_GPL(wakeup_source_register); @@ -185,12 +235,69 @@ void wakeup_source_unregister(struct wakeup_source *ws) { if (ws) { wakeup_source_remove(ws); + if (ws->dev) + wakeup_source_sysfs_remove(ws); + wakeup_source_destroy(ws); } } EXPORT_SYMBOL_GPL(wakeup_source_unregister); /** + * wakeup_sources_read_lock - Lock wakeup source list for read. + * + * Returns an index of srcu lock for struct wakeup_srcu. + * This index must be passed to the matching wakeup_sources_read_unlock(). + */ +int wakeup_sources_read_lock(void) +{ + return srcu_read_lock(&wakeup_srcu); +} +EXPORT_SYMBOL_GPL(wakeup_sources_read_lock); + +/** + * wakeup_sources_read_unlock - Unlock wakeup source list. + * @idx: return value from corresponding wakeup_sources_read_lock() + */ +void wakeup_sources_read_unlock(int idx) +{ + srcu_read_unlock(&wakeup_srcu, idx); +} +EXPORT_SYMBOL_GPL(wakeup_sources_read_unlock); + +/** + * wakeup_sources_walk_start - Begin a walk on wakeup source list + * + * Returns first object of the list of wakeup sources. + * + * Note that to be safe, wakeup sources list needs to be locked by calling + * wakeup_source_read_lock() for this. + */ +struct wakeup_source *wakeup_sources_walk_start(void) +{ + struct list_head *ws_head = &wakeup_sources; + + return list_entry_rcu(ws_head->next, struct wakeup_source, entry); +} +EXPORT_SYMBOL_GPL(wakeup_sources_walk_start); + +/** + * wakeup_sources_walk_next - Get next wakeup source from the list + * @ws: Previous wakeup source object + * + * Note that to be safe, wakeup sources list needs to be locked by calling + * wakeup_source_read_lock() for this. + */ +struct wakeup_source *wakeup_sources_walk_next(struct wakeup_source *ws) +{ + struct list_head *ws_head = &wakeup_sources; + + return list_next_or_null_rcu(ws_head, &ws->entry, + struct wakeup_source, entry); +} +EXPORT_SYMBOL_GPL(wakeup_sources_walk_next); + +/** * device_wakeup_attach - Attach a wakeup source object to a device object. * @dev: Device to handle. * @ws: Wakeup source object to attach to @dev. @@ -205,6 +312,8 @@ static int device_wakeup_attach(struct device *dev, struct wakeup_source *ws) return -EEXIST; } dev->power.wakeup = ws; + if (dev->power.wakeirq) + device_wakeup_attach_irq(dev, dev->power.wakeirq); spin_unlock_irq(&dev->power.lock); return 0; } @@ -223,7 +332,10 @@ int device_wakeup_enable(struct device *dev) if (!dev || !dev->power.can_wakeup) return -EINVAL; - ws = wakeup_source_register(dev_name(dev)); + if (pm_sleep_transition_in_progress()) + dev_dbg(dev, "Suspicious %s() during system transition!\n", __func__); + + ws = wakeup_source_register(dev, dev_name(dev)); if (!ws) return -ENOMEM; @@ -236,6 +348,81 @@ int device_wakeup_enable(struct device *dev) EXPORT_SYMBOL_GPL(device_wakeup_enable); /** + * device_wakeup_attach_irq - Attach a wakeirq to a wakeup source + * @dev: Device to handle + * @wakeirq: Device specific wakeirq entry + * + * Attach a device wakeirq to the wakeup source so the device + * wake IRQ can be configured automatically for suspend and + * resume. + * + * Call under the device's power.lock lock. + */ +void device_wakeup_attach_irq(struct device *dev, + struct wake_irq *wakeirq) +{ + struct wakeup_source *ws; + + ws = dev->power.wakeup; + if (!ws) + return; + + if (ws->wakeirq) + dev_err(dev, "Leftover wakeup IRQ found, overriding\n"); + + ws->wakeirq = wakeirq; +} + +/** + * device_wakeup_detach_irq - Detach a wakeirq from a wakeup source + * @dev: Device to handle + * + * Removes a device wakeirq from the wakeup source. + * + * Call under the device's power.lock lock. + */ +void device_wakeup_detach_irq(struct device *dev) +{ + struct wakeup_source *ws; + + ws = dev->power.wakeup; + if (ws) + ws->wakeirq = NULL; +} + +/** + * device_wakeup_arm_wake_irqs - + * + * Iterates over the list of device wakeirqs to arm them. + */ +void device_wakeup_arm_wake_irqs(void) +{ + struct wakeup_source *ws; + int srcuidx; + + srcuidx = srcu_read_lock(&wakeup_srcu); + list_for_each_entry_rcu_locked(ws, &wakeup_sources, entry) + dev_pm_arm_wake_irq(ws->wakeirq); + srcu_read_unlock(&wakeup_srcu, srcuidx); +} + +/** + * device_wakeup_disarm_wake_irqs - + * + * Iterates over the list of device wakeirqs to disarm them. + */ +void device_wakeup_disarm_wake_irqs(void) +{ + struct wakeup_source *ws; + int srcuidx; + + srcuidx = srcu_read_lock(&wakeup_srcu); + list_for_each_entry_rcu_locked(ws, &wakeup_sources, entry) + dev_pm_disarm_wake_irq(ws->wakeirq); + srcu_read_unlock(&wakeup_srcu, srcuidx); +} + +/** * device_wakeup_detach - Detach a device's wakeup source object from it. * @dev: Device to detach the wakeup source object from. * @@ -259,18 +446,15 @@ static struct wakeup_source *device_wakeup_detach(struct device *dev) * Detach the @dev's wakeup source object from it, unregister this wakeup source * object and destroy it. */ -int device_wakeup_disable(struct device *dev) +void device_wakeup_disable(struct device *dev) { struct wakeup_source *ws; if (!dev || !dev->power.can_wakeup) - return -EINVAL; + return; ws = device_wakeup_detach(dev); - if (ws) - wakeup_source_unregister(ws); - - return 0; + wakeup_source_unregister(ws); } EXPORT_SYMBOL_GPL(device_wakeup_disable); @@ -291,56 +475,47 @@ void device_set_wakeup_capable(struct device *dev, bool capable) if (!!dev->power.can_wakeup == !!capable) return; + dev->power.can_wakeup = capable; if (device_is_registered(dev) && !list_empty(&dev->power.entry)) { if (capable) { - if (wakeup_sysfs_add(dev)) - return; + int ret = wakeup_sysfs_add(dev); + + if (ret) + dev_info(dev, "Wakeup sysfs attributes not added\n"); } else { wakeup_sysfs_remove(dev); } } - dev->power.can_wakeup = capable; } EXPORT_SYMBOL_GPL(device_set_wakeup_capable); /** - * device_init_wakeup - Device wakeup initialization. + * device_set_wakeup_enable - Enable or disable a device to wake up the system. * @dev: Device to handle. - * @enable: Whether or not to enable @dev as a wakeup device. - * - * By default, most devices should leave wakeup disabled. The exceptions are - * devices that everyone expects to be wakeup sources: keyboards, power buttons, - * possibly network interfaces, etc. Also, devices that don't generate their - * own wakeup requests but merely forward requests from one bus to another - * (like PCI bridges) should have wakeup enabled by default. + * @enable: enable/disable flag */ -int device_init_wakeup(struct device *dev, bool enable) +int device_set_wakeup_enable(struct device *dev, bool enable) { - int ret = 0; - - if (enable) { - device_set_wakeup_capable(dev, true); - ret = device_wakeup_enable(dev); - } else { - device_set_wakeup_capable(dev, false); - } + if (enable) + return device_wakeup_enable(dev); - return ret; + device_wakeup_disable(dev); + return 0; } -EXPORT_SYMBOL_GPL(device_init_wakeup); +EXPORT_SYMBOL_GPL(device_set_wakeup_enable); /** - * device_set_wakeup_enable - Enable or disable a device to wake up the system. - * @dev: Device to handle. + * wakeup_source_not_usable - validate the given wakeup source. + * @ws: Wakeup source to be validated. */ -int device_set_wakeup_enable(struct device *dev, bool enable) +static bool wakeup_source_not_usable(struct wakeup_source *ws) { - if (!dev || !dev->power.can_wakeup) - return -EINVAL; - - return enable ? device_wakeup_enable(dev) : device_wakeup_disable(dev); + /* + * Use the timer struct to check if the given wakeup source has been + * initialized by wakeup_source_add() and it is not going away. + */ + return ws->timer.function != pm_wakeup_timer_fn; } -EXPORT_SYMBOL_GPL(device_set_wakeup_enable); /* * The functions below use the observation that each wakeup event starts a @@ -371,22 +546,19 @@ EXPORT_SYMBOL_GPL(device_set_wakeup_enable); */ /** - * wakup_source_activate - Mark given wakeup source as active. + * wakeup_source_activate - Mark given wakeup source as active. * @ws: Wakeup source to handle. * * Update the @ws' statistics and, if @ws has just been activated, notify the PM - * core of the event by incrementing the counter of of wakeup events being + * core of the event by incrementing the counter of the wakeup events being * processed. */ static void wakeup_source_activate(struct wakeup_source *ws) { unsigned int cec; - /* - * active wakeup source should bring the system - * out of PM_SUSPEND_FREEZE state - */ - freeze_wake(); + if (WARN_ONCE(wakeup_source_not_usable(ws), "unusable wakeup source\n")) + return; ws->active = true; ws->active_count++; @@ -403,8 +575,9 @@ static void wakeup_source_activate(struct wakeup_source *ws) /** * wakeup_source_report_event - Report wakeup event using the given source. * @ws: Wakeup source to report the event for. + * @hard: If set, abort suspends in progress and wake up from suspend-to-idle. */ -static void wakeup_source_report_event(struct wakeup_source *ws) +static void wakeup_source_report_event(struct wakeup_source *ws, bool hard) { ws->event_count++; /* This is racy, but the counter is approximate anyway. */ @@ -413,6 +586,9 @@ static void wakeup_source_report_event(struct wakeup_source *ws) if (!ws->active) wakeup_source_activate(ws); + + if (hard) + pm_system_wakeup(); } /** @@ -430,8 +606,8 @@ void __pm_stay_awake(struct wakeup_source *ws) spin_lock_irqsave(&ws->lock, flags); - wakeup_source_report_event(ws); - del_timer(&ws->timer); + wakeup_source_report_event(ws, false); + timer_delete(&ws->timer); ws->timer_expires = 0; spin_unlock_irqrestore(&ws->lock, flags); @@ -474,7 +650,7 @@ static inline void update_prevent_sleep_time(struct wakeup_source *ws, #endif /** - * wakup_source_deactivate - Mark given wakeup source as inactive. + * wakeup_source_deactivate - Mark given wakeup source as inactive. * @ws: Wakeup source to handle. * * Update the @ws' statistics and notify the PM core that the wakeup source has @@ -511,7 +687,7 @@ static void wakeup_source_deactivate(struct wakeup_source *ws) ws->max_time = duration; ws->last_time = now; - del_timer(&ws->timer); + timer_delete(&ws->timer); ws->timer_expires = 0; if (ws->autosleep_enabled) @@ -519,7 +695,7 @@ static void wakeup_source_deactivate(struct wakeup_source *ws) /* * Increment the counter of registered wakeup events and decrement the - * couter of wakeup events in progress simultaneously. + * counter of wakeup events in progress simultaneously. */ cec = atomic_add_return(MAX_IN_PROGRESS, &combined_event_count); trace_wakeup_source_deactivate(ws->name, cec); @@ -573,15 +749,15 @@ EXPORT_SYMBOL_GPL(pm_relax); /** * pm_wakeup_timer_fn - Delayed finalization of a wakeup event. - * @data: Address of the wakeup source object associated with the event source. + * @t: timer list * * Call wakeup_source_deactivate() for the wakeup source whose address is stored * in @data if it is currently active and its timer has not been canceled and * the expiration time of the timer is not in future. */ -static void pm_wakeup_timer_fn(unsigned long data) +static void pm_wakeup_timer_fn(struct timer_list *t) { - struct wakeup_source *ws = (struct wakeup_source *)data; + struct wakeup_source *ws = timer_container_of(ws, t, timer); unsigned long flags; spin_lock_irqsave(&ws->lock, flags); @@ -596,9 +772,10 @@ static void pm_wakeup_timer_fn(unsigned long data) } /** - * __pm_wakeup_event - Notify the PM core of a wakeup event. + * pm_wakeup_ws_event - Notify the PM core of a wakeup event. * @ws: Wakeup source object associated with the event source. * @msec: Anticipated event processing time (in milliseconds). + * @hard: If set, abort suspends in progress and wake up from suspend-to-idle. * * Notify the PM core of a wakeup event whose source is @ws that will take * approximately @msec milliseconds to be processed by the kernel. If @ws is @@ -607,7 +784,7 @@ static void pm_wakeup_timer_fn(unsigned long data) * * It is safe to call this function from interrupt context. */ -void __pm_wakeup_event(struct wakeup_source *ws, unsigned int msec) +void pm_wakeup_ws_event(struct wakeup_source *ws, unsigned int msec, bool hard) { unsigned long flags; unsigned long expires; @@ -617,7 +794,7 @@ void __pm_wakeup_event(struct wakeup_source *ws, unsigned int msec) spin_lock_irqsave(&ws->lock, flags); - wakeup_source_report_event(ws); + wakeup_source_report_event(ws, hard); if (!msec) { wakeup_source_deactivate(ws); @@ -636,17 +813,17 @@ void __pm_wakeup_event(struct wakeup_source *ws, unsigned int msec) unlock: spin_unlock_irqrestore(&ws->lock, flags); } -EXPORT_SYMBOL_GPL(__pm_wakeup_event); - +EXPORT_SYMBOL_GPL(pm_wakeup_ws_event); /** - * pm_wakeup_event - Notify the PM core of a wakeup event. + * pm_wakeup_dev_event - Notify the PM core of a wakeup event. * @dev: Device the wakeup event is related to. * @msec: Anticipated event processing time (in milliseconds). + * @hard: If set, abort suspends in progress and wake up from suspend-to-idle. * - * Call __pm_wakeup_event() for the @dev's wakeup source object. + * Call pm_wakeup_ws_event() for the @dev's wakeup source object. */ -void pm_wakeup_event(struct device *dev, unsigned int msec) +void pm_wakeup_dev_event(struct device *dev, unsigned int msec, bool hard) { unsigned long flags; @@ -654,21 +831,21 @@ void pm_wakeup_event(struct device *dev, unsigned int msec) return; spin_lock_irqsave(&dev->power.lock, flags); - __pm_wakeup_event(dev->power.wakeup, msec); + pm_wakeup_ws_event(dev->power.wakeup, msec, hard); spin_unlock_irqrestore(&dev->power.lock, flags); } -EXPORT_SYMBOL_GPL(pm_wakeup_event); +EXPORT_SYMBOL_GPL(pm_wakeup_dev_event); void pm_print_active_wakeup_sources(void) { struct wakeup_source *ws; - int active = 0; + int srcuidx, active = 0; struct wakeup_source *last_activity_ws = NULL; - rcu_read_lock(); - list_for_each_entry_rcu(ws, &wakeup_sources, entry) { + srcuidx = srcu_read_lock(&wakeup_srcu); + list_for_each_entry_rcu_locked(ws, &wakeup_sources, entry) { if (ws->active) { - pr_info("active wakeup source: %s\n", ws->name); + pm_pr_dbg("active wakeup source: %s\n", ws->name); active = 1; } else if (!active && (!last_activity_ws || @@ -679,9 +856,9 @@ void pm_print_active_wakeup_sources(void) } if (!active && last_activity_ws) - pr_info("last active wakeup source: %s\n", + pm_pr_dbg("last active wakeup source: %s\n", last_activity_ws->name); - rcu_read_unlock(); + srcu_read_unlock(&wakeup_srcu, srcuidx); } EXPORT_SYMBOL_GPL(pm_print_active_wakeup_sources); @@ -698,7 +875,7 @@ bool pm_wakeup_pending(void) unsigned long flags; bool ret = false; - spin_lock_irqsave(&events_lock, flags); + raw_spin_lock_irqsave(&events_lock, flags); if (events_check_enabled) { unsigned int cnt, inpr; @@ -706,14 +883,70 @@ bool pm_wakeup_pending(void) ret = (cnt != saved_count || inpr > 0); events_check_enabled = !ret; } - spin_unlock_irqrestore(&events_lock, flags); + raw_spin_unlock_irqrestore(&events_lock, flags); if (ret) { - pr_info("PM: Wakeup pending, aborting suspend\n"); + pm_pr_dbg("Wakeup pending, aborting suspend\n"); pm_print_active_wakeup_sources(); } - return ret; + return ret || atomic_read(&pm_abort_suspend) > 0; +} +EXPORT_SYMBOL_GPL(pm_wakeup_pending); + +void pm_system_wakeup(void) +{ + atomic_inc(&pm_abort_suspend); + s2idle_wake(); +} +EXPORT_SYMBOL_GPL(pm_system_wakeup); + +void pm_system_cancel_wakeup(void) +{ + atomic_dec_if_positive(&pm_abort_suspend); +} + +void pm_wakeup_clear(unsigned int irq_number) +{ + raw_spin_lock_irq(&wakeup_irq_lock); + + if (irq_number && wakeup_irq[0] == irq_number) + wakeup_irq[0] = wakeup_irq[1]; + else + wakeup_irq[0] = 0; + + wakeup_irq[1] = 0; + + raw_spin_unlock_irq(&wakeup_irq_lock); + + if (!irq_number) + atomic_set(&pm_abort_suspend, 0); +} + +void pm_system_irq_wakeup(unsigned int irq_number) +{ + unsigned long flags; + + raw_spin_lock_irqsave(&wakeup_irq_lock, flags); + + if (wakeup_irq[0] == 0) + wakeup_irq[0] = irq_number; + else if (wakeup_irq[1] == 0) + wakeup_irq[1] = irq_number; + else + irq_number = 0; + + pm_pr_dbg("Triggering wakeup from IRQ %d\n", irq_number); + + raw_spin_unlock_irqrestore(&wakeup_irq_lock, flags); + + if (irq_number) + pm_system_wakeup(); +} + +unsigned int pm_wakeup_irq(void) +{ + return wakeup_irq[0]; } /** @@ -741,7 +974,7 @@ bool pm_get_wakeup_count(unsigned int *count, bool block) split_counters(&cnt, &inpr); if (inpr == 0 || signal_pending(current)) break; - + pm_print_active_wakeup_sources(); schedule(); } finish_wait(&wakeup_count_wait_queue, &wait); @@ -768,28 +1001,29 @@ bool pm_save_wakeup_count(unsigned int count) unsigned long flags; events_check_enabled = false; - spin_lock_irqsave(&events_lock, flags); + raw_spin_lock_irqsave(&events_lock, flags); split_counters(&cnt, &inpr); if (cnt == count && inpr == 0) { saved_count = count; events_check_enabled = true; } - spin_unlock_irqrestore(&events_lock, flags); + raw_spin_unlock_irqrestore(&events_lock, flags); return events_check_enabled; } #ifdef CONFIG_PM_AUTOSLEEP /** * pm_wakep_autosleep_enabled - Modify autosleep_enabled for all wakeup sources. - * @enabled: Whether to set or to clear the autosleep_enabled flags. + * @set: Whether to set or to clear the autosleep_enabled flags. */ void pm_wakep_autosleep_enabled(bool set) { struct wakeup_source *ws; ktime_t now = ktime_get(); + int srcuidx; - rcu_read_lock(); - list_for_each_entry_rcu(ws, &wakeup_sources, entry) { + srcuidx = srcu_read_lock(&wakeup_srcu); + list_for_each_entry_rcu_locked(ws, &wakeup_sources, entry) { spin_lock_irq(&ws->lock); if (ws->autosleep_enabled != set) { ws->autosleep_enabled = set; @@ -802,12 +1036,10 @@ void pm_wakep_autosleep_enabled(bool set) } spin_unlock_irq(&ws->lock); } - rcu_read_unlock(); + srcu_read_unlock(&wakeup_srcu, srcuidx); } #endif /* CONFIG_PM_AUTOSLEEP */ -static struct dentry *wakeup_sources_stats_dentry; - /** * print_wakeup_source_stats - Print wakeup source statistics information. * @m: seq_file to print the statistics into. @@ -822,7 +1054,6 @@ static int print_wakeup_source_stats(struct seq_file *m, unsigned long active_count; ktime_t active_time; ktime_t prevent_sleep_time; - int ret; spin_lock_irqsave(&ws->lock, flags); @@ -835,52 +1066,100 @@ static int print_wakeup_source_stats(struct seq_file *m, active_time = ktime_sub(now, ws->last_time); total_time = ktime_add(total_time, active_time); - if (active_time.tv64 > max_time.tv64) + if (active_time > max_time) max_time = active_time; if (ws->autosleep_enabled) prevent_sleep_time = ktime_add(prevent_sleep_time, ktime_sub(now, ws->start_prevent_time)); } else { - active_time = ktime_set(0, 0); + active_time = 0; } - ret = seq_printf(m, "%-12s\t%lu\t\t%lu\t\t%lu\t\t%lu\t\t" - "%lld\t\t%lld\t\t%lld\t\t%lld\t\t%lld\n", - ws->name, active_count, ws->event_count, - ws->wakeup_count, ws->expire_count, - ktime_to_ms(active_time), ktime_to_ms(total_time), - ktime_to_ms(max_time), ktime_to_ms(ws->last_time), - ktime_to_ms(prevent_sleep_time)); + seq_printf(m, "%-12s\t%lu\t\t%lu\t\t%lu\t\t%lu\t\t%lld\t\t%lld\t\t%lld\t\t%lld\t\t%lld\n", + ws->name, active_count, ws->event_count, + ws->wakeup_count, ws->expire_count, + ktime_to_ms(active_time), ktime_to_ms(total_time), + ktime_to_ms(max_time), ktime_to_ms(ws->last_time), + ktime_to_ms(prevent_sleep_time)); spin_unlock_irqrestore(&ws->lock, flags); - return ret; + return 0; +} + +static void *wakeup_sources_stats_seq_start(struct seq_file *m, + loff_t *pos) +{ + struct wakeup_source *ws; + loff_t n = *pos; + int *srcuidx = m->private; + + if (n == 0) { + seq_puts(m, "name\t\tactive_count\tevent_count\twakeup_count\t" + "expire_count\tactive_since\ttotal_time\tmax_time\t" + "last_change\tprevent_suspend_time\n"); + } + + *srcuidx = srcu_read_lock(&wakeup_srcu); + list_for_each_entry_rcu_locked(ws, &wakeup_sources, entry) { + if (n-- <= 0) + return ws; + } + + return NULL; +} + +static void *wakeup_sources_stats_seq_next(struct seq_file *m, + void *v, loff_t *pos) +{ + struct wakeup_source *ws = v; + struct wakeup_source *next_ws = NULL; + + ++(*pos); + + list_for_each_entry_continue_rcu(ws, &wakeup_sources, entry) { + next_ws = ws; + break; + } + + if (!next_ws) + print_wakeup_source_stats(m, &deleted_ws); + + return next_ws; +} + +static void wakeup_sources_stats_seq_stop(struct seq_file *m, void *v) +{ + int *srcuidx = m->private; + + srcu_read_unlock(&wakeup_srcu, *srcuidx); } /** - * wakeup_sources_stats_show - Print wakeup sources statistics information. + * wakeup_sources_stats_seq_show - Print wakeup sources statistics information. * @m: seq_file to print the statistics into. + * @v: wakeup_source of each iteration */ -static int wakeup_sources_stats_show(struct seq_file *m, void *unused) +static int wakeup_sources_stats_seq_show(struct seq_file *m, void *v) { - struct wakeup_source *ws; + struct wakeup_source *ws = v; - seq_puts(m, "name\t\tactive_count\tevent_count\twakeup_count\t" - "expire_count\tactive_since\ttotal_time\tmax_time\t" - "last_change\tprevent_suspend_time\n"); - - rcu_read_lock(); - list_for_each_entry_rcu(ws, &wakeup_sources, entry) - print_wakeup_source_stats(m, ws); - rcu_read_unlock(); + print_wakeup_source_stats(m, ws); return 0; } +static const struct seq_operations wakeup_sources_stats_seq_ops = { + .start = wakeup_sources_stats_seq_start, + .next = wakeup_sources_stats_seq_next, + .stop = wakeup_sources_stats_seq_stop, + .show = wakeup_sources_stats_seq_show, +}; + static int wakeup_sources_stats_open(struct inode *inode, struct file *file) { - return single_open(file, wakeup_sources_stats_show, NULL); + return seq_open_private(file, &wakeup_sources_stats_seq_ops, sizeof(int)); } static const struct file_operations wakeup_sources_stats_fops = { @@ -888,13 +1167,13 @@ static const struct file_operations wakeup_sources_stats_fops = { .open = wakeup_sources_stats_open, .read = seq_read, .llseek = seq_lseek, - .release = single_release, + .release = seq_release_private, }; static int __init wakeup_sources_debugfs_init(void) { - wakeup_sources_stats_dentry = debugfs_create_file("wakeup_sources", - S_IRUGO, NULL, NULL, &wakeup_sources_stats_fops); + debugfs_create_file("wakeup_sources", 0444, NULL, NULL, + &wakeup_sources_stats_fops); return 0; } diff --git a/drivers/base/power/wakeup_stats.c b/drivers/base/power/wakeup_stats.c new file mode 100644 index 000000000000..3ffd427248e8 --- /dev/null +++ b/drivers/base/power/wakeup_stats.c @@ -0,0 +1,219 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Wakeup statistics in sysfs + * + * Copyright (c) 2019 Linux Foundation + * Copyright (c) 2019 Greg Kroah-Hartman <gregkh@linuxfoundation.org> + * Copyright (c) 2019 Google Inc. + */ + +#include <linux/device.h> +#include <linux/idr.h> +#include <linux/init.h> +#include <linux/kdev_t.h> +#include <linux/kernel.h> +#include <linux/kobject.h> +#include <linux/slab.h> +#include <linux/timekeeping.h> + +#include "power.h" + +static struct class *wakeup_class; + +#define wakeup_attr(_name) \ +static ssize_t _name##_show(struct device *dev, \ + struct device_attribute *attr, char *buf) \ +{ \ + struct wakeup_source *ws = dev_get_drvdata(dev); \ + \ + return sysfs_emit(buf, "%lu\n", ws->_name); \ +} \ +static DEVICE_ATTR_RO(_name) + +wakeup_attr(active_count); +wakeup_attr(event_count); +wakeup_attr(wakeup_count); +wakeup_attr(expire_count); +wakeup_attr(relax_count); + +static ssize_t active_time_ms_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct wakeup_source *ws = dev_get_drvdata(dev); + ktime_t active_time = + ws->active ? ktime_sub(ktime_get(), ws->last_time) : 0; + + return sysfs_emit(buf, "%lld\n", ktime_to_ms(active_time)); +} +static DEVICE_ATTR_RO(active_time_ms); + +static ssize_t total_time_ms_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct wakeup_source *ws = dev_get_drvdata(dev); + ktime_t active_time; + ktime_t total_time = ws->total_time; + + if (ws->active) { + active_time = ktime_sub(ktime_get(), ws->last_time); + total_time = ktime_add(total_time, active_time); + } + + return sysfs_emit(buf, "%lld\n", ktime_to_ms(total_time)); +} +static DEVICE_ATTR_RO(total_time_ms); + +static ssize_t max_time_ms_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct wakeup_source *ws = dev_get_drvdata(dev); + ktime_t active_time; + ktime_t max_time = ws->max_time; + + if (ws->active) { + active_time = ktime_sub(ktime_get(), ws->last_time); + if (active_time > max_time) + max_time = active_time; + } + + return sysfs_emit(buf, "%lld\n", ktime_to_ms(max_time)); +} +static DEVICE_ATTR_RO(max_time_ms); + +static ssize_t last_change_ms_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct wakeup_source *ws = dev_get_drvdata(dev); + + return sysfs_emit(buf, "%lld\n", ktime_to_ms(ws->last_time)); +} +static DEVICE_ATTR_RO(last_change_ms); + +static ssize_t name_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct wakeup_source *ws = dev_get_drvdata(dev); + + return sysfs_emit(buf, "%s\n", ws->name); +} +static DEVICE_ATTR_RO(name); + +static ssize_t prevent_suspend_time_ms_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct wakeup_source *ws = dev_get_drvdata(dev); + ktime_t prevent_sleep_time = ws->prevent_sleep_time; + + if (ws->active && ws->autosleep_enabled) { + prevent_sleep_time = ktime_add(prevent_sleep_time, + ktime_sub(ktime_get(), ws->start_prevent_time)); + } + + return sysfs_emit(buf, "%lld\n", ktime_to_ms(prevent_sleep_time)); +} +static DEVICE_ATTR_RO(prevent_suspend_time_ms); + +static struct attribute *wakeup_source_attrs[] = { + &dev_attr_name.attr, + &dev_attr_active_count.attr, + &dev_attr_event_count.attr, + &dev_attr_wakeup_count.attr, + &dev_attr_expire_count.attr, + &dev_attr_relax_count.attr, + &dev_attr_active_time_ms.attr, + &dev_attr_total_time_ms.attr, + &dev_attr_max_time_ms.attr, + &dev_attr_last_change_ms.attr, + &dev_attr_prevent_suspend_time_ms.attr, + NULL, +}; +ATTRIBUTE_GROUPS(wakeup_source); + +static void device_create_release(struct device *dev) +{ + kfree(dev); +} + +static struct device *wakeup_source_device_create(struct device *parent, + struct wakeup_source *ws) +{ + struct device *dev = NULL; + int retval; + + dev = kzalloc(sizeof(*dev), GFP_KERNEL); + if (!dev) { + retval = -ENOMEM; + goto error; + } + + device_initialize(dev); + dev->devt = MKDEV(0, 0); + dev->class = wakeup_class; + dev->parent = parent; + dev->groups = wakeup_source_groups; + dev->release = device_create_release; + dev_set_drvdata(dev, ws); + device_set_pm_not_required(dev); + + retval = dev_set_name(dev, "wakeup%d", ws->id); + if (retval) + goto error; + + retval = device_add(dev); + if (retval) + goto error; + + return dev; + +error: + put_device(dev); + return ERR_PTR(retval); +} + +/** + * wakeup_source_sysfs_add - Add wakeup_source attributes to sysfs. + * @parent: Device given wakeup source is associated with (or NULL if virtual). + * @ws: Wakeup source to be added in sysfs. + */ +int wakeup_source_sysfs_add(struct device *parent, struct wakeup_source *ws) +{ + struct device *dev; + + dev = wakeup_source_device_create(parent, ws); + if (IS_ERR(dev)) + return PTR_ERR(dev); + ws->dev = dev; + + return 0; +} + +/** + * pm_wakeup_source_sysfs_add - Add wakeup_source attributes to sysfs + * for a device if they're missing. + * @parent: Device given wakeup source is associated with + */ +int pm_wakeup_source_sysfs_add(struct device *parent) +{ + if (!parent->power.wakeup || parent->power.wakeup->dev) + return 0; + + return wakeup_source_sysfs_add(parent, parent->power.wakeup); +} + +/** + * wakeup_source_sysfs_remove - Remove wakeup_source attributes from sysfs. + * @ws: Wakeup source to be removed from sysfs. + */ +void wakeup_source_sysfs_remove(struct wakeup_source *ws) +{ + device_unregister(ws->dev); +} + +static int __init wakeup_sources_sysfs_init(void) +{ + wakeup_class = class_create("wakeup"); + + return PTR_ERR_OR_ZERO(wakeup_class); +} +postcore_initcall(wakeup_sources_sysfs_init); diff --git a/drivers/base/property.c b/drivers/base/property.c new file mode 100644 index 000000000000..6a63860579dd --- /dev/null +++ b/drivers/base/property.c @@ -0,0 +1,1482 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * property.c - Unified device property interface. + * + * Copyright (C) 2014, Intel Corporation + * Authors: Rafael J. Wysocki <rafael.j.wysocki@intel.com> + * Mika Westerberg <mika.westerberg@linux.intel.com> + */ + +#include <linux/device.h> +#include <linux/err.h> +#include <linux/export.h> +#include <linux/kconfig.h> +#include <linux/of.h> +#include <linux/property.h> +#include <linux/phy.h> +#include <linux/slab.h> +#include <linux/string.h> +#include <linux/types.h> + +struct fwnode_handle *__dev_fwnode(struct device *dev) +{ + return IS_ENABLED(CONFIG_OF) && dev->of_node ? + of_fwnode_handle(dev->of_node) : dev->fwnode; +} +EXPORT_SYMBOL_GPL(__dev_fwnode); + +const struct fwnode_handle *__dev_fwnode_const(const struct device *dev) +{ + return IS_ENABLED(CONFIG_OF) && dev->of_node ? + of_fwnode_handle(dev->of_node) : dev->fwnode; +} +EXPORT_SYMBOL_GPL(__dev_fwnode_const); + +/** + * device_property_present - check if a property of a device is present + * @dev: Device whose property is being checked + * @propname: Name of the property + * + * Check if property @propname is present in the device firmware description. + * + * Return: true if property @propname is present. Otherwise, returns false. + */ +bool device_property_present(const struct device *dev, const char *propname) +{ + return fwnode_property_present(dev_fwnode(dev), propname); +} +EXPORT_SYMBOL_GPL(device_property_present); + +/** + * fwnode_property_present - check if a property of a firmware node is present + * @fwnode: Firmware node whose property to check + * @propname: Name of the property + * + * Return: true if property @propname is present. Otherwise, returns false. + */ +bool fwnode_property_present(const struct fwnode_handle *fwnode, + const char *propname) +{ + bool ret; + + if (IS_ERR_OR_NULL(fwnode)) + return false; + + ret = fwnode_call_bool_op(fwnode, property_present, propname); + if (ret) + return ret; + + return fwnode_call_bool_op(fwnode->secondary, property_present, propname); +} +EXPORT_SYMBOL_GPL(fwnode_property_present); + +/** + * device_property_read_bool - Return the value for a boolean property of a device + * @dev: Device whose property is being checked + * @propname: Name of the property + * + * Return if property @propname is true or false in the device firmware description. + * + * Return: true if property @propname is present. Otherwise, returns false. + */ +bool device_property_read_bool(const struct device *dev, const char *propname) +{ + return fwnode_property_read_bool(dev_fwnode(dev), propname); +} +EXPORT_SYMBOL_GPL(device_property_read_bool); + +/** + * fwnode_property_read_bool - Return the value for a boolean property of a firmware node + * @fwnode: Firmware node whose property to check + * @propname: Name of the property + * + * Return if property @propname is true or false in the firmware description. + */ +bool fwnode_property_read_bool(const struct fwnode_handle *fwnode, + const char *propname) +{ + bool ret; + + if (IS_ERR_OR_NULL(fwnode)) + return false; + + ret = fwnode_call_bool_op(fwnode, property_read_bool, propname); + if (ret) + return ret; + + return fwnode_call_bool_op(fwnode->secondary, property_read_bool, propname); +} +EXPORT_SYMBOL_GPL(fwnode_property_read_bool); + +/** + * device_property_read_u8_array - return a u8 array property of a device + * @dev: Device to get the property of + * @propname: Name of the property + * @val: The values are stored here or %NULL to return the number of values + * @nval: Size of the @val array + * + * Function reads an array of u8 properties with @propname from the device + * firmware description and stores them to @val if found. + * + * It's recommended to call device_property_count_u8() instead of calling + * this function with @val equals %NULL and @nval equals 0. + * + * Return: number of values if @val was %NULL, + * %0 if the property was found (success), + * %-EINVAL if given arguments are not valid, + * %-ENODATA if the property does not have a value, + * %-EPROTO if the property is not an array of numbers, + * %-EOVERFLOW if the size of the property is not as expected. + * %-ENXIO if no suitable firmware interface is present. + */ +int device_property_read_u8_array(const struct device *dev, const char *propname, + u8 *val, size_t nval) +{ + return fwnode_property_read_u8_array(dev_fwnode(dev), propname, val, nval); +} +EXPORT_SYMBOL_GPL(device_property_read_u8_array); + +/** + * device_property_read_u16_array - return a u16 array property of a device + * @dev: Device to get the property of + * @propname: Name of the property + * @val: The values are stored here or %NULL to return the number of values + * @nval: Size of the @val array + * + * Function reads an array of u16 properties with @propname from the device + * firmware description and stores them to @val if found. + * + * It's recommended to call device_property_count_u16() instead of calling + * this function with @val equals %NULL and @nval equals 0. + * + * Return: number of values if @val was %NULL, + * %0 if the property was found (success), + * %-EINVAL if given arguments are not valid, + * %-ENODATA if the property does not have a value, + * %-EPROTO if the property is not an array of numbers, + * %-EOVERFLOW if the size of the property is not as expected. + * %-ENXIO if no suitable firmware interface is present. + */ +int device_property_read_u16_array(const struct device *dev, const char *propname, + u16 *val, size_t nval) +{ + return fwnode_property_read_u16_array(dev_fwnode(dev), propname, val, nval); +} +EXPORT_SYMBOL_GPL(device_property_read_u16_array); + +/** + * device_property_read_u32_array - return a u32 array property of a device + * @dev: Device to get the property of + * @propname: Name of the property + * @val: The values are stored here or %NULL to return the number of values + * @nval: Size of the @val array + * + * Function reads an array of u32 properties with @propname from the device + * firmware description and stores them to @val if found. + * + * It's recommended to call device_property_count_u32() instead of calling + * this function with @val equals %NULL and @nval equals 0. + * + * Return: number of values if @val was %NULL, + * %0 if the property was found (success), + * %-EINVAL if given arguments are not valid, + * %-ENODATA if the property does not have a value, + * %-EPROTO if the property is not an array of numbers, + * %-EOVERFLOW if the size of the property is not as expected. + * %-ENXIO if no suitable firmware interface is present. + */ +int device_property_read_u32_array(const struct device *dev, const char *propname, + u32 *val, size_t nval) +{ + return fwnode_property_read_u32_array(dev_fwnode(dev), propname, val, nval); +} +EXPORT_SYMBOL_GPL(device_property_read_u32_array); + +/** + * device_property_read_u64_array - return a u64 array property of a device + * @dev: Device to get the property of + * @propname: Name of the property + * @val: The values are stored here or %NULL to return the number of values + * @nval: Size of the @val array + * + * Function reads an array of u64 properties with @propname from the device + * firmware description and stores them to @val if found. + * + * It's recommended to call device_property_count_u64() instead of calling + * this function with @val equals %NULL and @nval equals 0. + * + * Return: number of values if @val was %NULL, + * %0 if the property was found (success), + * %-EINVAL if given arguments are not valid, + * %-ENODATA if the property does not have a value, + * %-EPROTO if the property is not an array of numbers, + * %-EOVERFLOW if the size of the property is not as expected. + * %-ENXIO if no suitable firmware interface is present. + */ +int device_property_read_u64_array(const struct device *dev, const char *propname, + u64 *val, size_t nval) +{ + return fwnode_property_read_u64_array(dev_fwnode(dev), propname, val, nval); +} +EXPORT_SYMBOL_GPL(device_property_read_u64_array); + +/** + * device_property_read_string_array - return a string array property of device + * @dev: Device to get the property of + * @propname: Name of the property + * @val: The values are stored here or %NULL to return the number of values + * @nval: Size of the @val array + * + * Function reads an array of string properties with @propname from the device + * firmware description and stores them to @val if found. + * + * It's recommended to call device_property_string_array_count() instead of calling + * this function with @val equals %NULL and @nval equals 0. + * + * Return: number of values read on success if @val is non-NULL, + * number of values available on success if @val is NULL, + * %-EINVAL if given arguments are not valid, + * %-ENODATA if the property does not have a value, + * %-EPROTO or %-EILSEQ if the property is not an array of strings, + * %-EOVERFLOW if the size of the property is not as expected. + * %-ENXIO if no suitable firmware interface is present. + */ +int device_property_read_string_array(const struct device *dev, const char *propname, + const char **val, size_t nval) +{ + return fwnode_property_read_string_array(dev_fwnode(dev), propname, val, nval); +} +EXPORT_SYMBOL_GPL(device_property_read_string_array); + +/** + * device_property_read_string - return a string property of a device + * @dev: Device to get the property of + * @propname: Name of the property + * @val: The value is stored here + * + * Function reads property @propname from the device firmware description and + * stores the value into @val if found. The value is checked to be a string. + * + * Return: %0 if the property was found (success), + * %-EINVAL if given arguments are not valid, + * %-ENODATA if the property does not have a value, + * %-EPROTO or %-EILSEQ if the property type is not a string. + * %-ENXIO if no suitable firmware interface is present. + */ +int device_property_read_string(const struct device *dev, const char *propname, + const char **val) +{ + return fwnode_property_read_string(dev_fwnode(dev), propname, val); +} +EXPORT_SYMBOL_GPL(device_property_read_string); + +/** + * device_property_match_string - find a string in an array and return index + * @dev: Device to get the property of + * @propname: Name of the property holding the array + * @string: String to look for + * + * Find a given string in a string array and if it is found return the + * index back. + * + * Return: index, starting from %0, if the property was found (success), + * %-EINVAL if given arguments are not valid, + * %-ENODATA if the property does not have a value, + * %-EPROTO if the property is not an array of strings, + * %-ENXIO if no suitable firmware interface is present. + */ +int device_property_match_string(const struct device *dev, const char *propname, + const char *string) +{ + return fwnode_property_match_string(dev_fwnode(dev), propname, string); +} +EXPORT_SYMBOL_GPL(device_property_match_string); + +static int fwnode_property_read_int_array(const struct fwnode_handle *fwnode, + const char *propname, + unsigned int elem_size, void *val, + size_t nval) +{ + int ret; + + if (IS_ERR_OR_NULL(fwnode)) + return -EINVAL; + + ret = fwnode_call_int_op(fwnode, property_read_int_array, propname, + elem_size, val, nval); + if (ret != -EINVAL) + return ret; + + return fwnode_call_int_op(fwnode->secondary, property_read_int_array, propname, + elem_size, val, nval); +} + +/** + * fwnode_property_read_u8_array - return a u8 array property of firmware node + * @fwnode: Firmware node to get the property of + * @propname: Name of the property + * @val: The values are stored here or %NULL to return the number of values + * @nval: Size of the @val array + * + * Read an array of u8 properties with @propname from @fwnode and stores them to + * @val if found. + * + * It's recommended to call fwnode_property_count_u8() instead of calling + * this function with @val equals %NULL and @nval equals 0. + * + * Return: number of values if @val was %NULL, + * %0 if the property was found (success), + * %-EINVAL if given arguments are not valid, + * %-ENODATA if the property does not have a value, + * %-EPROTO if the property is not an array of numbers, + * %-EOVERFLOW if the size of the property is not as expected, + * %-ENXIO if no suitable firmware interface is present. + */ +int fwnode_property_read_u8_array(const struct fwnode_handle *fwnode, + const char *propname, u8 *val, size_t nval) +{ + return fwnode_property_read_int_array(fwnode, propname, sizeof(u8), + val, nval); +} +EXPORT_SYMBOL_GPL(fwnode_property_read_u8_array); + +/** + * fwnode_property_read_u16_array - return a u16 array property of firmware node + * @fwnode: Firmware node to get the property of + * @propname: Name of the property + * @val: The values are stored here or %NULL to return the number of values + * @nval: Size of the @val array + * + * Read an array of u16 properties with @propname from @fwnode and store them to + * @val if found. + * + * It's recommended to call fwnode_property_count_u16() instead of calling + * this function with @val equals %NULL and @nval equals 0. + * + * Return: number of values if @val was %NULL, + * %0 if the property was found (success), + * %-EINVAL if given arguments are not valid, + * %-ENODATA if the property does not have a value, + * %-EPROTO if the property is not an array of numbers, + * %-EOVERFLOW if the size of the property is not as expected, + * %-ENXIO if no suitable firmware interface is present. + */ +int fwnode_property_read_u16_array(const struct fwnode_handle *fwnode, + const char *propname, u16 *val, size_t nval) +{ + return fwnode_property_read_int_array(fwnode, propname, sizeof(u16), + val, nval); +} +EXPORT_SYMBOL_GPL(fwnode_property_read_u16_array); + +/** + * fwnode_property_read_u32_array - return a u32 array property of firmware node + * @fwnode: Firmware node to get the property of + * @propname: Name of the property + * @val: The values are stored here or %NULL to return the number of values + * @nval: Size of the @val array + * + * Read an array of u32 properties with @propname from @fwnode store them to + * @val if found. + * + * It's recommended to call fwnode_property_count_u32() instead of calling + * this function with @val equals %NULL and @nval equals 0. + * + * Return: number of values if @val was %NULL, + * %0 if the property was found (success), + * %-EINVAL if given arguments are not valid, + * %-ENODATA if the property does not have a value, + * %-EPROTO if the property is not an array of numbers, + * %-EOVERFLOW if the size of the property is not as expected, + * %-ENXIO if no suitable firmware interface is present. + */ +int fwnode_property_read_u32_array(const struct fwnode_handle *fwnode, + const char *propname, u32 *val, size_t nval) +{ + return fwnode_property_read_int_array(fwnode, propname, sizeof(u32), + val, nval); +} +EXPORT_SYMBOL_GPL(fwnode_property_read_u32_array); + +/** + * fwnode_property_read_u64_array - return a u64 array property firmware node + * @fwnode: Firmware node to get the property of + * @propname: Name of the property + * @val: The values are stored here or %NULL to return the number of values + * @nval: Size of the @val array + * + * Read an array of u64 properties with @propname from @fwnode and store them to + * @val if found. + * + * It's recommended to call fwnode_property_count_u64() instead of calling + * this function with @val equals %NULL and @nval equals 0. + * + * Return: number of values if @val was %NULL, + * %0 if the property was found (success), + * %-EINVAL if given arguments are not valid, + * %-ENODATA if the property does not have a value, + * %-EPROTO if the property is not an array of numbers, + * %-EOVERFLOW if the size of the property is not as expected, + * %-ENXIO if no suitable firmware interface is present. + */ +int fwnode_property_read_u64_array(const struct fwnode_handle *fwnode, + const char *propname, u64 *val, size_t nval) +{ + return fwnode_property_read_int_array(fwnode, propname, sizeof(u64), + val, nval); +} +EXPORT_SYMBOL_GPL(fwnode_property_read_u64_array); + +/** + * fwnode_property_read_string_array - return string array property of a node + * @fwnode: Firmware node to get the property of + * @propname: Name of the property + * @val: The values are stored here or %NULL to return the number of values + * @nval: Size of the @val array + * + * Read an string list property @propname from the given firmware node and store + * them to @val if found. + * + * It's recommended to call fwnode_property_string_array_count() instead of calling + * this function with @val equals %NULL and @nval equals 0. + * + * Return: number of values read on success if @val is non-NULL, + * number of values available on success if @val is NULL, + * %-EINVAL if given arguments are not valid, + * %-ENODATA if the property does not have a value, + * %-EPROTO or %-EILSEQ if the property is not an array of strings, + * %-EOVERFLOW if the size of the property is not as expected, + * %-ENXIO if no suitable firmware interface is present. + */ +int fwnode_property_read_string_array(const struct fwnode_handle *fwnode, + const char *propname, const char **val, + size_t nval) +{ + int ret; + + if (IS_ERR_OR_NULL(fwnode)) + return -EINVAL; + + ret = fwnode_call_int_op(fwnode, property_read_string_array, propname, + val, nval); + if (ret != -EINVAL) + return ret; + + return fwnode_call_int_op(fwnode->secondary, property_read_string_array, propname, + val, nval); +} +EXPORT_SYMBOL_GPL(fwnode_property_read_string_array); + +/** + * fwnode_property_read_string - return a string property of a firmware node + * @fwnode: Firmware node to get the property of + * @propname: Name of the property + * @val: The value is stored here + * + * Read property @propname from the given firmware node and store the value into + * @val if found. The value is checked to be a string. + * + * Return: %0 if the property was found (success), + * %-EINVAL if given arguments are not valid, + * %-ENODATA if the property does not have a value, + * %-EPROTO or %-EILSEQ if the property is not a string, + * %-ENXIO if no suitable firmware interface is present. + */ +int fwnode_property_read_string(const struct fwnode_handle *fwnode, + const char *propname, const char **val) +{ + int ret = fwnode_property_read_string_array(fwnode, propname, val, 1); + + return ret < 0 ? ret : 0; +} +EXPORT_SYMBOL_GPL(fwnode_property_read_string); + +/** + * fwnode_property_match_string - find a string in an array and return index + * @fwnode: Firmware node to get the property of + * @propname: Name of the property holding the array + * @string: String to look for + * + * Find a given string in a string array and if it is found return the + * index back. + * + * Return: index, starting from %0, if the property was found (success), + * %-EINVAL if given arguments are not valid, + * %-ENODATA if the property does not have a value, + * %-EPROTO if the property is not an array of strings, + * %-ENXIO if no suitable firmware interface is present. + */ +int fwnode_property_match_string(const struct fwnode_handle *fwnode, + const char *propname, const char *string) +{ + const char **values; + int nval, ret; + + nval = fwnode_property_string_array_count(fwnode, propname); + if (nval < 0) + return nval; + + if (nval == 0) + return -ENODATA; + + values = kcalloc(nval, sizeof(*values), GFP_KERNEL); + if (!values) + return -ENOMEM; + + ret = fwnode_property_read_string_array(fwnode, propname, values, nval); + if (ret < 0) + goto out_free; + + ret = match_string(values, nval, string); + if (ret < 0) + ret = -ENODATA; + +out_free: + kfree(values); + return ret; +} +EXPORT_SYMBOL_GPL(fwnode_property_match_string); + +/** + * fwnode_property_match_property_string - find a property string value in an array and return index + * @fwnode: Firmware node to get the property of + * @propname: Name of the property holding the string value + * @array: String array to search in + * @n: Size of the @array + * + * Find a property string value in a given @array and if it is found return + * the index back. + * + * Return: index, starting from %0, if the string value was found in the @array (success), + * %-ENOENT when the string value was not found in the @array, + * %-EINVAL if given arguments are not valid, + * %-ENODATA if the property does not have a value, + * %-EPROTO or %-EILSEQ if the property is not a string, + * %-ENXIO if no suitable firmware interface is present. + */ +int fwnode_property_match_property_string(const struct fwnode_handle *fwnode, + const char *propname, const char * const *array, size_t n) +{ + const char *string; + int ret; + + ret = fwnode_property_read_string(fwnode, propname, &string); + if (ret) + return ret; + + ret = match_string(array, n, string); + if (ret < 0) + ret = -ENOENT; + + return ret; +} +EXPORT_SYMBOL_GPL(fwnode_property_match_property_string); + +/** + * fwnode_property_get_reference_args() - Find a reference with arguments + * @fwnode: Firmware node where to look for the reference + * @prop: The name of the property + * @nargs_prop: The name of the property telling the number of + * arguments in the referred node. NULL if @nargs is known, + * otherwise @nargs is ignored. + * @nargs: Number of arguments. Ignored if @nargs_prop is non-NULL. + * @index: Index of the reference, from zero onwards. + * @args: Result structure with reference and integer arguments. + * May be NULL. + * + * Obtain a reference based on a named property in an fwnode, with + * integer arguments. + * + * The caller is responsible for calling fwnode_handle_put() on the returned + * @args->fwnode pointer. + * + * Return: %0 on success + * %-ENOENT when the index is out of bounds, the index has an empty + * reference or the property was not found + * %-EINVAL on parse error + */ +int fwnode_property_get_reference_args(const struct fwnode_handle *fwnode, + const char *prop, const char *nargs_prop, + unsigned int nargs, unsigned int index, + struct fwnode_reference_args *args) +{ + int ret; + + if (IS_ERR_OR_NULL(fwnode)) + return -ENOENT; + + ret = fwnode_call_int_op(fwnode, get_reference_args, prop, nargs_prop, + nargs, index, args); + if (ret == 0) + return ret; + + if (IS_ERR_OR_NULL(fwnode->secondary)) + return ret; + + return fwnode_call_int_op(fwnode->secondary, get_reference_args, prop, nargs_prop, + nargs, index, args); +} +EXPORT_SYMBOL_GPL(fwnode_property_get_reference_args); + +/** + * fwnode_find_reference - Find named reference to a fwnode_handle + * @fwnode: Firmware node where to look for the reference + * @name: The name of the reference + * @index: Index of the reference + * + * @index can be used when the named reference holds a table of references. + * + * The caller is responsible for calling fwnode_handle_put() on the returned + * fwnode pointer. + * + * Return: a pointer to the reference fwnode, when found. Otherwise, + * returns an error pointer. + */ +struct fwnode_handle *fwnode_find_reference(const struct fwnode_handle *fwnode, + const char *name, + unsigned int index) +{ + struct fwnode_reference_args args; + int ret; + + ret = fwnode_property_get_reference_args(fwnode, name, NULL, 0, index, + &args); + return ret ? ERR_PTR(ret) : args.fwnode; +} +EXPORT_SYMBOL_GPL(fwnode_find_reference); + +/** + * fwnode_get_name - Return the name of a node + * @fwnode: The firmware node + * + * Return: a pointer to the node name, or %NULL. + */ +const char *fwnode_get_name(const struct fwnode_handle *fwnode) +{ + return fwnode_call_ptr_op(fwnode, get_name); +} +EXPORT_SYMBOL_GPL(fwnode_get_name); + +/** + * fwnode_get_name_prefix - Return the prefix of node for printing purposes + * @fwnode: The firmware node + * + * Return: the prefix of a node, intended to be printed right before the node. + * The prefix works also as a separator between the nodes. + */ +const char *fwnode_get_name_prefix(const struct fwnode_handle *fwnode) +{ + return fwnode_call_ptr_op(fwnode, get_name_prefix); +} + +/** + * fwnode_name_eq - Return true if node name is equal + * @fwnode: The firmware node + * @name: The name to which to compare the node name + * + * Compare the name provided as an argument to the name of the node, stopping + * the comparison at either NUL or '@' character, whichever comes first. This + * function is generally used for comparing node names while ignoring the + * possible unit address of the node. + * + * Return: true if the node name matches with the name provided in the @name + * argument, false otherwise. + */ +bool fwnode_name_eq(const struct fwnode_handle *fwnode, const char *name) +{ + const char *node_name; + ptrdiff_t len; + + node_name = fwnode_get_name(fwnode); + if (!node_name) + return false; + + len = strchrnul(node_name, '@') - node_name; + + return str_has_prefix(node_name, name) == len; +} +EXPORT_SYMBOL_GPL(fwnode_name_eq); + +/** + * fwnode_get_parent - Return parent firwmare node + * @fwnode: Firmware whose parent is retrieved + * + * The caller is responsible for calling fwnode_handle_put() on the returned + * fwnode pointer. + * + * Return: parent firmware node of the given node if possible or %NULL if no + * parent was available. + */ +struct fwnode_handle *fwnode_get_parent(const struct fwnode_handle *fwnode) +{ + return fwnode_call_ptr_op(fwnode, get_parent); +} +EXPORT_SYMBOL_GPL(fwnode_get_parent); + +/** + * fwnode_get_next_parent - Iterate to the node's parent + * @fwnode: Firmware whose parent is retrieved + * + * This is like fwnode_get_parent() except that it drops the refcount + * on the passed node, making it suitable for iterating through a + * node's parents. + * + * The caller is responsible for calling fwnode_handle_put() on the returned + * fwnode pointer. Note that this function also puts a reference to @fwnode + * unconditionally. + * + * Return: parent firmware node of the given node if possible or %NULL if no + * parent was available. + */ +struct fwnode_handle *fwnode_get_next_parent(struct fwnode_handle *fwnode) +{ + struct fwnode_handle *parent = fwnode_get_parent(fwnode); + + fwnode_handle_put(fwnode); + + return parent; +} +EXPORT_SYMBOL_GPL(fwnode_get_next_parent); + +/** + * fwnode_count_parents - Return the number of parents a node has + * @fwnode: The node the parents of which are to be counted + * + * Return: the number of parents a node has. + */ +unsigned int fwnode_count_parents(const struct fwnode_handle *fwnode) +{ + struct fwnode_handle *parent; + unsigned int count = 0; + + fwnode_for_each_parent_node(fwnode, parent) + count++; + + return count; +} +EXPORT_SYMBOL_GPL(fwnode_count_parents); + +/** + * fwnode_get_nth_parent - Return an nth parent of a node + * @fwnode: The node the parent of which is requested + * @depth: Distance of the parent from the node + * + * The caller is responsible for calling fwnode_handle_put() on the returned + * fwnode pointer. + * + * Return: the nth parent of a node. If there is no parent at the requested + * @depth, %NULL is returned. If @depth is 0, the functionality is equivalent to + * fwnode_handle_get(). For @depth == 1, it is fwnode_get_parent() and so on. + */ +struct fwnode_handle *fwnode_get_nth_parent(struct fwnode_handle *fwnode, + unsigned int depth) +{ + struct fwnode_handle *parent; + + if (depth == 0) + return fwnode_handle_get(fwnode); + + fwnode_for_each_parent_node(fwnode, parent) { + if (--depth == 0) + return parent; + } + return NULL; +} +EXPORT_SYMBOL_GPL(fwnode_get_nth_parent); + +/** + * fwnode_get_next_child_node - Return the next child node handle for a node + * @fwnode: Firmware node to find the next child node for. + * @child: Handle to one of the node's child nodes or a %NULL handle. + * + * The caller is responsible for calling fwnode_handle_put() on the returned + * fwnode pointer. Note that this function also puts a reference to @child + * unconditionally. + */ +struct fwnode_handle * +fwnode_get_next_child_node(const struct fwnode_handle *fwnode, + struct fwnode_handle *child) +{ + return fwnode_call_ptr_op(fwnode, get_next_child_node, child); +} +EXPORT_SYMBOL_GPL(fwnode_get_next_child_node); + +/** + * fwnode_get_next_available_child_node - Return the next available child node handle for a node + * @fwnode: Firmware node to find the next child node for. + * @child: Handle to one of the node's child nodes or a %NULL handle. + * + * The caller is responsible for calling fwnode_handle_put() on the returned + * fwnode pointer. Note that this function also puts a reference to @child + * unconditionally. + */ +struct fwnode_handle * +fwnode_get_next_available_child_node(const struct fwnode_handle *fwnode, + struct fwnode_handle *child) +{ + struct fwnode_handle *next_child = child; + + if (IS_ERR_OR_NULL(fwnode)) + return NULL; + + do { + next_child = fwnode_get_next_child_node(fwnode, next_child); + if (!next_child) + return NULL; + } while (!fwnode_device_is_available(next_child)); + + return next_child; +} +EXPORT_SYMBOL_GPL(fwnode_get_next_available_child_node); + +/** + * device_get_next_child_node - Return the next child node handle for a device + * @dev: Device to find the next child node for. + * @child: Handle to one of the device's child nodes or a %NULL handle. + * + * The caller is responsible for calling fwnode_handle_put() on the returned + * fwnode pointer. Note that this function also puts a reference to @child + * unconditionally. + */ +struct fwnode_handle *device_get_next_child_node(const struct device *dev, + struct fwnode_handle *child) +{ + const struct fwnode_handle *fwnode = dev_fwnode(dev); + struct fwnode_handle *next; + + if (IS_ERR_OR_NULL(fwnode)) + return NULL; + + /* Try to find a child in primary fwnode */ + next = fwnode_get_next_child_node(fwnode, child); + if (next) + return next; + + /* When no more children in primary, continue with secondary */ + return fwnode_get_next_child_node(fwnode->secondary, child); +} +EXPORT_SYMBOL_GPL(device_get_next_child_node); + +/** + * fwnode_get_named_child_node - Return first matching named child node handle + * @fwnode: Firmware node to find the named child node for. + * @childname: String to match child node name against. + * + * The caller is responsible for calling fwnode_handle_put() on the returned + * fwnode pointer. + */ +struct fwnode_handle * +fwnode_get_named_child_node(const struct fwnode_handle *fwnode, + const char *childname) +{ + return fwnode_call_ptr_op(fwnode, get_named_child_node, childname); +} +EXPORT_SYMBOL_GPL(fwnode_get_named_child_node); + +/** + * device_get_named_child_node - Return first matching named child node handle + * @dev: Device to find the named child node for. + * @childname: String to match child node name against. + * + * The caller is responsible for calling fwnode_handle_put() on the returned + * fwnode pointer. + */ +struct fwnode_handle *device_get_named_child_node(const struct device *dev, + const char *childname) +{ + return fwnode_get_named_child_node(dev_fwnode(dev), childname); +} +EXPORT_SYMBOL_GPL(device_get_named_child_node); + +/** + * fwnode_handle_get - Obtain a reference to a device node + * @fwnode: Pointer to the device node to obtain the reference to. + * + * The caller is responsible for calling fwnode_handle_put() on the returned + * fwnode pointer. + * + * Return: the fwnode handle. + */ +struct fwnode_handle *fwnode_handle_get(struct fwnode_handle *fwnode) +{ + if (!fwnode_has_op(fwnode, get)) + return fwnode; + + return fwnode_call_ptr_op(fwnode, get); +} +EXPORT_SYMBOL_GPL(fwnode_handle_get); + +/** + * fwnode_device_is_available - check if a device is available for use + * @fwnode: Pointer to the fwnode of the device. + * + * Return: true if device is available for use. Otherwise, returns false. + * + * For fwnode node types that don't implement the .device_is_available() + * operation, this function returns true. + */ +bool fwnode_device_is_available(const struct fwnode_handle *fwnode) +{ + if (IS_ERR_OR_NULL(fwnode)) + return false; + + if (!fwnode_has_op(fwnode, device_is_available)) + return true; + + return fwnode_call_bool_op(fwnode, device_is_available); +} +EXPORT_SYMBOL_GPL(fwnode_device_is_available); + +/** + * fwnode_get_child_node_count - return the number of child nodes for a given firmware node + * @fwnode: Pointer to the parent firmware node + * + * Return: the number of child nodes for a given firmware node. + */ +unsigned int fwnode_get_child_node_count(const struct fwnode_handle *fwnode) +{ + struct fwnode_handle *child; + unsigned int count = 0; + + fwnode_for_each_child_node(fwnode, child) + count++; + + return count; +} +EXPORT_SYMBOL_GPL(fwnode_get_child_node_count); + +/** + * fwnode_get_named_child_node_count - number of child nodes with given name + * @fwnode: Node which child nodes are counted. + * @name: String to match child node name against. + * + * Scan child nodes and count all the nodes with a specific name. Potential + * 'number' -ending after the 'at sign' for scanned names is ignored. + * E.g.:: + * fwnode_get_named_child_node_count(fwnode, "channel"); + * would match all the nodes:: + * channel { }, channel@0 {}, channel@0xabba {}... + * + * Return: the number of child nodes with a matching name for a given device. + */ +unsigned int fwnode_get_named_child_node_count(const struct fwnode_handle *fwnode, + const char *name) +{ + struct fwnode_handle *child; + unsigned int count = 0; + + fwnode_for_each_named_child_node(fwnode, child, name) + count++; + + return count; +} +EXPORT_SYMBOL_GPL(fwnode_get_named_child_node_count); + +bool device_dma_supported(const struct device *dev) +{ + return fwnode_call_bool_op(dev_fwnode(dev), device_dma_supported); +} +EXPORT_SYMBOL_GPL(device_dma_supported); + +enum dev_dma_attr device_get_dma_attr(const struct device *dev) +{ + if (!fwnode_has_op(dev_fwnode(dev), device_get_dma_attr)) + return DEV_DMA_NOT_SUPPORTED; + + return fwnode_call_int_op(dev_fwnode(dev), device_get_dma_attr); +} +EXPORT_SYMBOL_GPL(device_get_dma_attr); + +/** + * fwnode_get_phy_mode - Get phy mode for given firmware node + * @fwnode: Pointer to the given node + * + * The function gets phy interface string from property 'phy-mode' or + * 'phy-connection-type', and return its index in phy_modes table, or errno in + * error case. + */ +int fwnode_get_phy_mode(const struct fwnode_handle *fwnode) +{ + const char *pm; + int err, i; + + err = fwnode_property_read_string(fwnode, "phy-mode", &pm); + if (err < 0) + err = fwnode_property_read_string(fwnode, + "phy-connection-type", &pm); + if (err < 0) + return err; + + for (i = 0; i < PHY_INTERFACE_MODE_MAX; i++) + if (!strcasecmp(pm, phy_modes(i))) + return i; + + return -ENODEV; +} +EXPORT_SYMBOL_GPL(fwnode_get_phy_mode); + +/** + * device_get_phy_mode - Get phy mode for given device + * @dev: Pointer to the given device + * + * The function gets phy interface string from property 'phy-mode' or + * 'phy-connection-type', and return its index in phy_modes table, or errno in + * error case. + */ +int device_get_phy_mode(struct device *dev) +{ + return fwnode_get_phy_mode(dev_fwnode(dev)); +} +EXPORT_SYMBOL_GPL(device_get_phy_mode); + +/** + * fwnode_iomap - Maps the memory mapped IO for a given fwnode + * @fwnode: Pointer to the firmware node + * @index: Index of the IO range + * + * Return: a pointer to the mapped memory. + */ +void __iomem *fwnode_iomap(struct fwnode_handle *fwnode, int index) +{ + return fwnode_call_ptr_op(fwnode, iomap, index); +} +EXPORT_SYMBOL(fwnode_iomap); + +/** + * fwnode_irq_get - Get IRQ directly from a fwnode + * @fwnode: Pointer to the firmware node + * @index: Zero-based index of the IRQ + * + * Return: Linux IRQ number on success. Negative errno on failure. + */ +int fwnode_irq_get(const struct fwnode_handle *fwnode, unsigned int index) +{ + int ret; + + ret = fwnode_call_int_op(fwnode, irq_get, index); + /* We treat mapping errors as invalid case */ + if (ret == 0) + return -EINVAL; + + return ret; +} +EXPORT_SYMBOL(fwnode_irq_get); + +/** + * fwnode_irq_get_byname - Get IRQ from a fwnode using its name + * @fwnode: Pointer to the firmware node + * @name: IRQ name + * + * Description: + * Find a match to the string @name in the 'interrupt-names' string array + * in _DSD for ACPI, or of_node for Device Tree. Then get the Linux IRQ + * number of the IRQ resource corresponding to the index of the matched + * string. + * + * Return: Linux IRQ number on success, or negative errno otherwise. + */ +int fwnode_irq_get_byname(const struct fwnode_handle *fwnode, const char *name) +{ + int index; + + if (!name) + return -EINVAL; + + index = fwnode_property_match_string(fwnode, "interrupt-names", name); + if (index < 0) + return index; + + return fwnode_irq_get(fwnode, index); +} +EXPORT_SYMBOL(fwnode_irq_get_byname); + +/** + * fwnode_graph_get_next_endpoint - Get next endpoint firmware node + * @fwnode: Pointer to the parent firmware node + * @prev: Previous endpoint node or %NULL to get the first + * + * The caller is responsible for calling fwnode_handle_put() on the returned + * fwnode pointer. Note that this function also puts a reference to @prev + * unconditionally. + * + * Return: an endpoint firmware node pointer or %NULL if no more endpoints + * are available. + */ +struct fwnode_handle * +fwnode_graph_get_next_endpoint(const struct fwnode_handle *fwnode, + struct fwnode_handle *prev) +{ + struct fwnode_handle *ep, *port_parent = NULL; + const struct fwnode_handle *parent; + + /* + * If this function is in a loop and the previous iteration returned + * an endpoint from fwnode->secondary, then we need to use the secondary + * as parent rather than @fwnode. + */ + if (prev) { + port_parent = fwnode_graph_get_port_parent(prev); + parent = port_parent; + } else { + parent = fwnode; + } + if (IS_ERR_OR_NULL(parent)) + return NULL; + + ep = fwnode_call_ptr_op(parent, graph_get_next_endpoint, prev); + if (ep) + goto out_put_port_parent; + + ep = fwnode_graph_get_next_endpoint(parent->secondary, NULL); + +out_put_port_parent: + fwnode_handle_put(port_parent); + return ep; +} +EXPORT_SYMBOL_GPL(fwnode_graph_get_next_endpoint); + +/** + * fwnode_graph_get_port_parent - Return the device fwnode of a port endpoint + * @endpoint: Endpoint firmware node of the port + * + * The caller is responsible for calling fwnode_handle_put() on the returned + * fwnode pointer. + * + * Return: the firmware node of the device the @endpoint belongs to. + */ +struct fwnode_handle * +fwnode_graph_get_port_parent(const struct fwnode_handle *endpoint) +{ + struct fwnode_handle *port, *parent; + + port = fwnode_get_parent(endpoint); + parent = fwnode_call_ptr_op(port, graph_get_port_parent); + + fwnode_handle_put(port); + + return parent; +} +EXPORT_SYMBOL_GPL(fwnode_graph_get_port_parent); + +/** + * fwnode_graph_get_remote_port_parent - Return fwnode of a remote device + * @fwnode: Endpoint firmware node pointing to the remote endpoint + * + * Extracts firmware node of a remote device the @fwnode points to. + * + * The caller is responsible for calling fwnode_handle_put() on the returned + * fwnode pointer. + */ +struct fwnode_handle * +fwnode_graph_get_remote_port_parent(const struct fwnode_handle *fwnode) +{ + struct fwnode_handle *endpoint, *parent; + + endpoint = fwnode_graph_get_remote_endpoint(fwnode); + parent = fwnode_graph_get_port_parent(endpoint); + + fwnode_handle_put(endpoint); + + return parent; +} +EXPORT_SYMBOL_GPL(fwnode_graph_get_remote_port_parent); + +/** + * fwnode_graph_get_remote_port - Return fwnode of a remote port + * @fwnode: Endpoint firmware node pointing to the remote endpoint + * + * Extracts firmware node of a remote port the @fwnode points to. + * + * The caller is responsible for calling fwnode_handle_put() on the returned + * fwnode pointer. + */ +struct fwnode_handle * +fwnode_graph_get_remote_port(const struct fwnode_handle *fwnode) +{ + return fwnode_get_next_parent(fwnode_graph_get_remote_endpoint(fwnode)); +} +EXPORT_SYMBOL_GPL(fwnode_graph_get_remote_port); + +/** + * fwnode_graph_get_remote_endpoint - Return fwnode of a remote endpoint + * @fwnode: Endpoint firmware node pointing to the remote endpoint + * + * Extracts firmware node of a remote endpoint the @fwnode points to. + * + * The caller is responsible for calling fwnode_handle_put() on the returned + * fwnode pointer. + */ +struct fwnode_handle * +fwnode_graph_get_remote_endpoint(const struct fwnode_handle *fwnode) +{ + return fwnode_call_ptr_op(fwnode, graph_get_remote_endpoint); +} +EXPORT_SYMBOL_GPL(fwnode_graph_get_remote_endpoint); + +static bool fwnode_graph_remote_available(struct fwnode_handle *ep) +{ + struct fwnode_handle *dev_node; + bool available; + + dev_node = fwnode_graph_get_remote_port_parent(ep); + available = fwnode_device_is_available(dev_node); + fwnode_handle_put(dev_node); + + return available; +} + +/** + * fwnode_graph_get_endpoint_by_id - get endpoint by port and endpoint numbers + * @fwnode: parent fwnode_handle containing the graph + * @port: identifier of the port node + * @endpoint: identifier of the endpoint node under the port node + * @flags: fwnode lookup flags + * + * The caller is responsible for calling fwnode_handle_put() on the returned + * fwnode pointer. + * + * Return: the fwnode handle of the local endpoint corresponding the port and + * endpoint IDs or %NULL if not found. + * + * If FWNODE_GRAPH_ENDPOINT_NEXT is passed in @flags and the specified endpoint + * has not been found, look for the closest endpoint ID greater than the + * specified one and return the endpoint that corresponds to it, if present. + * + * Does not return endpoints that belong to disabled devices or endpoints that + * are unconnected, unless FWNODE_GRAPH_DEVICE_DISABLED is passed in @flags. + */ +struct fwnode_handle * +fwnode_graph_get_endpoint_by_id(const struct fwnode_handle *fwnode, + u32 port, u32 endpoint, unsigned long flags) +{ + struct fwnode_handle *ep, *best_ep = NULL; + unsigned int best_ep_id = 0; + bool endpoint_next = flags & FWNODE_GRAPH_ENDPOINT_NEXT; + bool enabled_only = !(flags & FWNODE_GRAPH_DEVICE_DISABLED); + + fwnode_graph_for_each_endpoint(fwnode, ep) { + struct fwnode_endpoint fwnode_ep = { 0 }; + int ret; + + if (enabled_only && !fwnode_graph_remote_available(ep)) + continue; + + ret = fwnode_graph_parse_endpoint(ep, &fwnode_ep); + if (ret < 0) + continue; + + if (fwnode_ep.port != port) + continue; + + if (fwnode_ep.id == endpoint) + return ep; + + if (!endpoint_next) + continue; + + /* + * If the endpoint that has just been found is not the first + * matching one and the ID of the one found previously is closer + * to the requested endpoint ID, skip it. + */ + if (fwnode_ep.id < endpoint || + (best_ep && best_ep_id < fwnode_ep.id)) + continue; + + fwnode_handle_put(best_ep); + best_ep = fwnode_handle_get(ep); + best_ep_id = fwnode_ep.id; + } + + return best_ep; +} +EXPORT_SYMBOL_GPL(fwnode_graph_get_endpoint_by_id); + +/** + * fwnode_graph_get_endpoint_count - Count endpoints on a device node + * @fwnode: The node related to a device + * @flags: fwnode lookup flags + * Count endpoints in a device node. + * + * If FWNODE_GRAPH_DEVICE_DISABLED flag is specified, also unconnected endpoints + * and endpoints connected to disabled devices are counted. + */ +unsigned int fwnode_graph_get_endpoint_count(const struct fwnode_handle *fwnode, + unsigned long flags) +{ + struct fwnode_handle *ep; + unsigned int count = 0; + + fwnode_graph_for_each_endpoint(fwnode, ep) { + if (flags & FWNODE_GRAPH_DEVICE_DISABLED || + fwnode_graph_remote_available(ep)) + count++; + } + + return count; +} +EXPORT_SYMBOL_GPL(fwnode_graph_get_endpoint_count); + +/** + * fwnode_graph_parse_endpoint - parse common endpoint node properties + * @fwnode: pointer to endpoint fwnode_handle + * @endpoint: pointer to the fwnode endpoint data structure + * + * Parse @fwnode representing a graph endpoint node and store the + * information in @endpoint. The caller must hold a reference to + * @fwnode. + */ +int fwnode_graph_parse_endpoint(const struct fwnode_handle *fwnode, + struct fwnode_endpoint *endpoint) +{ + memset(endpoint, 0, sizeof(*endpoint)); + + return fwnode_call_int_op(fwnode, graph_parse_endpoint, endpoint); +} +EXPORT_SYMBOL(fwnode_graph_parse_endpoint); + +const void *device_get_match_data(const struct device *dev) +{ + return fwnode_call_ptr_op(dev_fwnode(dev), device_get_match_data, dev); +} +EXPORT_SYMBOL_GPL(device_get_match_data); + +static unsigned int fwnode_graph_devcon_matches(const struct fwnode_handle *fwnode, + const char *con_id, void *data, + devcon_match_fn_t match, + void **matches, + unsigned int matches_len) +{ + struct fwnode_handle *node; + struct fwnode_handle *ep; + unsigned int count = 0; + void *ret; + + fwnode_graph_for_each_endpoint(fwnode, ep) { + if (matches && count >= matches_len) { + fwnode_handle_put(ep); + break; + } + + node = fwnode_graph_get_remote_port_parent(ep); + if (!fwnode_device_is_available(node)) { + fwnode_handle_put(node); + continue; + } + + ret = match(node, con_id, data); + fwnode_handle_put(node); + if (ret) { + if (matches) + matches[count] = ret; + count++; + } + } + return count; +} + +static unsigned int fwnode_devcon_matches(const struct fwnode_handle *fwnode, + const char *con_id, void *data, + devcon_match_fn_t match, + void **matches, + unsigned int matches_len) +{ + struct fwnode_handle *node; + unsigned int count = 0; + unsigned int i; + void *ret; + + for (i = 0; ; i++) { + if (matches && count >= matches_len) + break; + + node = fwnode_find_reference(fwnode, con_id, i); + if (IS_ERR(node)) + break; + + ret = match(node, NULL, data); + fwnode_handle_put(node); + if (ret) { + if (matches) + matches[count] = ret; + count++; + } + } + + return count; +} + +/** + * fwnode_connection_find_match - Find connection from a device node + * @fwnode: Device node with the connection + * @con_id: Identifier for the connection + * @data: Data for the match function + * @match: Function to check and convert the connection description + * + * Find a connection with unique identifier @con_id between @fwnode and another + * device node. @match will be used to convert the connection description to + * data the caller is expecting to be returned. + */ +void *fwnode_connection_find_match(const struct fwnode_handle *fwnode, + const char *con_id, void *data, + devcon_match_fn_t match) +{ + unsigned int count; + void *ret; + + if (!fwnode || !match) + return NULL; + + count = fwnode_graph_devcon_matches(fwnode, con_id, data, match, &ret, 1); + if (count) + return ret; + + count = fwnode_devcon_matches(fwnode, con_id, data, match, &ret, 1); + return count ? ret : NULL; +} +EXPORT_SYMBOL_GPL(fwnode_connection_find_match); + +/** + * fwnode_connection_find_matches - Find connections from a device node + * @fwnode: Device node with the connection + * @con_id: Identifier for the connection + * @data: Data for the match function + * @match: Function to check and convert the connection description + * @matches: (Optional) array of pointers to fill with matches + * @matches_len: Length of @matches + * + * Find up to @matches_len connections with unique identifier @con_id between + * @fwnode and other device nodes. @match will be used to convert the + * connection description to data the caller is expecting to be returned + * through the @matches array. + * + * If @matches is %NULL @matches_len is ignored and the total number of resolved + * matches is returned. + * + * Return: Number of matches resolved, or negative errno. + */ +int fwnode_connection_find_matches(const struct fwnode_handle *fwnode, + const char *con_id, void *data, + devcon_match_fn_t match, + void **matches, unsigned int matches_len) +{ + unsigned int count_graph; + unsigned int count_ref; + + if (!fwnode || !match) + return -EINVAL; + + count_graph = fwnode_graph_devcon_matches(fwnode, con_id, data, match, + matches, matches_len); + + if (matches) { + matches += count_graph; + matches_len -= count_graph; + } + + count_ref = fwnode_devcon_matches(fwnode, con_id, data, match, + matches, matches_len); + + return count_graph + count_ref; +} +EXPORT_SYMBOL_GPL(fwnode_connection_find_matches); diff --git a/drivers/base/regmap/Kconfig b/drivers/base/regmap/Kconfig index f0d30543fcce..ffb2ef488298 100644 --- a/drivers/base/regmap/Kconfig +++ b/drivers/base/regmap/Kconfig @@ -1,22 +1,93 @@ +# SPDX-License-Identifier: GPL-2.0 # Generic register map support. There are no user servicable options here, # this is an API intended to be used by other kernel subsystems. These # subsystems should select the appropriate symbols. config REGMAP - default y if (REGMAP_I2C || REGMAP_SPI || REGMAP_MMIO || REGMAP_IRQ) - select LZO_COMPRESS - select LZO_DECOMPRESS - select IRQ_DOMAIN if REGMAP_IRQ bool + default y if (REGMAP_I2C || REGMAP_SPI || REGMAP_SPMI || REGMAP_W1 || REGMAP_AC97 || REGMAP_MMIO || REGMAP_IRQ || REGMAP_SOUNDWIRE || REGMAP_SOUNDWIRE_MBQ || REGMAP_SCCB || REGMAP_I3C || REGMAP_SPI_AVMM || REGMAP_MDIO || REGMAP_FSI) + help + Enable support for the Register Map (regmap) access API. + + Usually, this option is automatically selected when needed. + However, you may want to enable it manually for running the regmap + KUnit tests. + + If unsure, say N. + +config REGMAP_KUNIT + tristate "KUnit tests for regmap" + depends on KUNIT && REGMAP + default KUNIT_ALL_TESTS + select REGMAP_RAM + +config REGMAP_BUILD + bool "Enable regmap build" + depends on KUNIT + select REGMAP + help + This option exists purely to allow the regmap KUnit tests to + be enabled without having to enable some driver that uses + regmap due to unfortunate issues with how KUnit tests are + normally enabled. + +config REGMAP_AC97 + tristate config REGMAP_I2C tristate + depends on I2C + +config REGMAP_SLIMBUS + tristate + depends on SLIMBUS config REGMAP_SPI tristate + depends on SPI + +config REGMAP_SPMI + tristate + depends on SPMI + +config REGMAP_W1 + tristate + depends on W1 + +config REGMAP_MDIO + tristate + select MDIO_BUS config REGMAP_MMIO tristate config REGMAP_IRQ bool + select IRQ_DOMAIN + +config REGMAP_RAM + tristate + +config REGMAP_SOUNDWIRE + tristate + depends on SOUNDWIRE + +config REGMAP_SOUNDWIRE_MBQ + tristate + depends on SOUNDWIRE + +config REGMAP_SCCB + tristate + depends on I2C + +config REGMAP_I3C + tristate + depends on I3C + +config REGMAP_SPI_AVMM + tristate + depends on SPI + +config REGMAP_FSI + tristate + depends on FSI diff --git a/drivers/base/regmap/Makefile b/drivers/base/regmap/Makefile index cf129980abd0..5fdd0845b45e 100644 --- a/drivers/base/regmap/Makefile +++ b/drivers/base/regmap/Makefile @@ -1,7 +1,24 @@ +# SPDX-License-Identifier: GPL-2.0 +# For include/trace/define_trace.h to include trace.h +CFLAGS_regmap.o := -I$(src) + obj-$(CONFIG_REGMAP) += regmap.o regcache.o -obj-$(CONFIG_REGMAP) += regcache-rbtree.o regcache-lzo.o regcache-flat.o +obj-$(CONFIG_REGMAP) += regcache-rbtree.o regcache-flat.o regcache-maple.o obj-$(CONFIG_DEBUG_FS) += regmap-debugfs.o +obj-$(CONFIG_REGMAP_KUNIT) += regmap-kunit.o +obj-$(CONFIG_REGMAP_AC97) += regmap-ac97.o obj-$(CONFIG_REGMAP_I2C) += regmap-i2c.o +obj-$(CONFIG_REGMAP_RAM) += regmap-ram.o regmap-raw-ram.o +obj-$(CONFIG_REGMAP_SLIMBUS) += regmap-slimbus.o obj-$(CONFIG_REGMAP_SPI) += regmap-spi.o +obj-$(CONFIG_REGMAP_SPMI) += regmap-spmi.o obj-$(CONFIG_REGMAP_MMIO) += regmap-mmio.o obj-$(CONFIG_REGMAP_IRQ) += regmap-irq.o +obj-$(CONFIG_REGMAP_W1) += regmap-w1.o +obj-$(CONFIG_REGMAP_SOUNDWIRE) += regmap-sdw.o +obj-$(CONFIG_REGMAP_SOUNDWIRE_MBQ) += regmap-sdw-mbq.o +obj-$(CONFIG_REGMAP_SCCB) += regmap-sccb.o +obj-$(CONFIG_REGMAP_I3C) += regmap-i3c.o +obj-$(CONFIG_REGMAP_SPI_AVMM) += regmap-spi-avmm.o +obj-$(CONFIG_REGMAP_MDIO) += regmap-mdio.o +obj-$(CONFIG_REGMAP_FSI) += regmap-fsi.o diff --git a/drivers/base/regmap/internal.h b/drivers/base/regmap/internal.h index 29c83160ca29..1477329410ec 100644 --- a/drivers/base/regmap/internal.h +++ b/drivers/base/regmap/internal.h @@ -1,18 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Register map access API internal header * * Copyright 2011 Wolfson Microelectronics plc * * Author: Mark Brown <broonie@opensource.wolfsonmicro.com> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #ifndef _REGMAP_INTERNAL_H #define _REGMAP_INTERNAL_H +#include <linux/device.h> #include <linux/regmap.h> #include <linux/fs.h> #include <linux/list.h> @@ -34,6 +32,7 @@ struct regmap_format { size_t reg_bytes; size_t pad_bytes; size_t val_bytes; + s8 reg_shift; void (*format_write)(struct regmap *map, unsigned int reg, unsigned int val); void (*format_reg)(void *buf, unsigned int reg, unsigned int shift); @@ -44,18 +43,28 @@ struct regmap_format { struct regmap_async { struct list_head list; - struct work_struct cleanup; struct regmap *map; void *work_buf; }; struct regmap { - struct mutex mutex; - spinlock_t spinlock; - unsigned long spinlock_flags; + union { + struct mutex mutex; + struct { + spinlock_t spinlock; + unsigned long spinlock_flags; + }; + struct { + raw_spinlock_t raw_spinlock; + unsigned long raw_spinlock_flags; + }; + }; + struct lock_class_key *lock_key; regmap_lock lock; regmap_unlock unlock; void *lock_arg; /* This is passed to lock/unlock functions */ + gfp_t alloc_flags; + unsigned int reg_base; struct device *dev; /* Device we do I/O on */ void *work_buf; /* Scratch buffer used to format I/O */ @@ -67,9 +76,12 @@ struct regmap { spinlock_t async_lock; wait_queue_head_t async_waitq; struct list_head async_list; + struct list_head async_free; int async_ret; + bool async; #ifdef CONFIG_DEBUG_FS + bool debugfs_disable; struct dentry *debugfs; const char *debugfs_name; @@ -82,26 +94,41 @@ struct regmap { #endif unsigned int max_register; + bool max_register_is_set; bool (*writeable_reg)(struct device *dev, unsigned int reg); bool (*readable_reg)(struct device *dev, unsigned int reg); bool (*volatile_reg)(struct device *dev, unsigned int reg); bool (*precious_reg)(struct device *dev, unsigned int reg); + bool (*writeable_noinc_reg)(struct device *dev, unsigned int reg); + bool (*readable_noinc_reg)(struct device *dev, unsigned int reg); const struct regmap_access_table *wr_table; const struct regmap_access_table *rd_table; const struct regmap_access_table *volatile_table; const struct regmap_access_table *precious_table; + const struct regmap_access_table *wr_noinc_table; + const struct regmap_access_table *rd_noinc_table; int (*reg_read)(void *context, unsigned int reg, unsigned int *val); int (*reg_write)(void *context, unsigned int reg, unsigned int val); + int (*reg_update_bits)(void *context, unsigned int reg, + unsigned int mask, unsigned int val); + /* Bulk read/write */ + int (*read)(void *context, const void *reg_buf, size_t reg_size, + void *val_buf, size_t val_size); + int (*write)(void *context, const void *data, size_t count); - bool defer_caching; - - u8 read_flag_mask; - u8 write_flag_mask; + unsigned long read_flag_mask; + unsigned long write_flag_mask; /* number of bits to (left) shift the reg value when formatting*/ int reg_shift; int reg_stride; + int reg_stride_order; + + bool defer_caching; + + /* If set, will always write field to HW. */ + bool force_write_field; /* regcache specific members */ const struct regcache_ops *cache_ops; @@ -117,28 +144,41 @@ struct regmap { unsigned int num_reg_defaults_raw; /* if set, only the cache is modified not the HW */ - u32 cache_only; + bool cache_only; /* if set, only the HW is modified not the cache */ - u32 cache_bypass; + bool cache_bypass; /* if set, remember to free reg_defaults_raw */ bool cache_free; struct reg_default *reg_defaults; const void *reg_defaults_raw; void *cache; - u32 cache_dirty; - - unsigned long *cache_present; - unsigned int cache_present_nbits; + /* if set, the cache contains newer data than the HW */ + bool cache_dirty; + /* if set, the HW registers are known to match map->reg_defaults */ + bool no_sync_defaults; - struct reg_default *patch; + struct reg_sequence *patch; int patch_regs; - /* if set, converts bulk rw to single rw */ - bool use_single_rw; + /* if set, the regmap core can sleep */ + bool can_sleep; + + /* if set, converts bulk read to single read */ + bool use_single_read; + /* if set, converts bulk write to single write */ + bool use_single_write; + /* if set, the device supports multi write mode */ + bool can_multi_write; + + /* if set, raw reads/writes are limited to this size */ + size_t max_raw_read; + size_t max_raw_write; struct rb_root range_tree; void *selector_work_buf; /* Scratch buffer used for selector */ + + struct hwspinlock *hwlock; }; struct regcache_ops { @@ -146,16 +186,23 @@ struct regcache_ops { enum regcache_type type; int (*init)(struct regmap *map); int (*exit)(struct regmap *map); + int (*populate)(struct regmap *map); +#ifdef CONFIG_DEBUG_FS + void (*debugfs_init)(struct regmap *map); +#endif int (*read)(struct regmap *map, unsigned int reg, unsigned int *value); int (*write)(struct regmap *map, unsigned int reg, unsigned int value); int (*sync)(struct regmap *map, unsigned int min, unsigned int max); int (*drop)(struct regmap *map, unsigned int min, unsigned int max); }; +bool regmap_cached(struct regmap *map, unsigned int reg); bool regmap_writeable(struct regmap *map, unsigned int reg); bool regmap_readable(struct regmap *map, unsigned int reg); bool regmap_volatile(struct regmap *map, unsigned int reg); bool regmap_precious(struct regmap *map, unsigned int reg); +bool regmap_writeable_noinc(struct regmap *map, unsigned int reg); +bool regmap_readable_noinc(struct regmap *map, unsigned int reg); int _regmap_write(struct regmap *map, unsigned int reg, unsigned int val); @@ -182,16 +229,26 @@ struct regmap_field { /* lsb */ unsigned int shift; unsigned int reg; + + unsigned int id_size; + unsigned int id_offset; }; #ifdef CONFIG_DEBUG_FS extern void regmap_debugfs_initcall(void); -extern void regmap_debugfs_init(struct regmap *map, const char *name); +extern void regmap_debugfs_init(struct regmap *map); extern void regmap_debugfs_exit(struct regmap *map); + +static inline void regmap_debugfs_disable(struct regmap *map) +{ + map->debugfs_disable = true; +} + #else static inline void regmap_debugfs_initcall(void) { } -static inline void regmap_debugfs_init(struct regmap *map, const char *name) { } +static inline void regmap_debugfs_init(struct regmap *map) { } static inline void regmap_debugfs_exit(struct regmap *map) { } +static inline void regmap_debugfs_disable(struct regmap *map) { } #endif /* regcache core declarations */ @@ -203,8 +260,11 @@ int regcache_write(struct regmap *map, unsigned int reg, unsigned int value); int regcache_sync(struct regmap *map); int regcache_sync_block(struct regmap *map, void *block, + unsigned long *cache_present, unsigned int block_base, unsigned int start, unsigned int end); +bool regcache_reg_needs_sync(struct regmap *map, unsigned int reg, + unsigned int val); static inline const void *regcache_get_val_addr(struct regmap *map, const void *base, @@ -215,27 +275,76 @@ static inline const void *regcache_get_val_addr(struct regmap *map, unsigned int regcache_get_val(struct regmap *map, const void *base, unsigned int idx); -bool regcache_set_val(struct regmap *map, void *base, unsigned int idx, +void regcache_set_val(struct regmap *map, void *base, unsigned int idx, unsigned int val); int regcache_lookup_reg(struct regmap *map, unsigned int reg); -int regcache_set_reg_present(struct regmap *map, unsigned int reg); - -static inline bool regcache_reg_present(struct regmap *map, unsigned int reg) -{ - if (!map->cache_present) - return true; - if (reg > map->cache_present_nbits) - return false; - return map->cache_present[BIT_WORD(reg)] & BIT_MASK(reg); -} +int regcache_sync_val(struct regmap *map, unsigned int reg, unsigned int val); int _regmap_raw_write(struct regmap *map, unsigned int reg, - const void *val, size_t val_len, bool async); + const void *val, size_t val_len, bool noinc); void regmap_async_complete_cb(struct regmap_async *async, int ret); +enum regmap_endian regmap_get_val_endian(struct device *dev, + const struct regmap_bus *bus, + const struct regmap_config *config); + +extern struct regcache_ops regcache_flat_sparse_ops; extern struct regcache_ops regcache_rbtree_ops; -extern struct regcache_ops regcache_lzo_ops; +extern struct regcache_ops regcache_maple_ops; extern struct regcache_ops regcache_flat_ops; +static inline const char *regmap_name(const struct regmap *map) +{ + if (map->dev) + return dev_name(map->dev); + + return map->name; +} + +static inline unsigned int regmap_get_offset(const struct regmap *map, + unsigned int index) +{ + if (map->reg_stride_order >= 0) + return index << map->reg_stride_order; + else + return index * map->reg_stride; +} + +static inline unsigned int regcache_get_index_by_order(const struct regmap *map, + unsigned int reg) +{ + return reg >> map->reg_stride_order; +} + +struct regmap_ram_data { + unsigned int *vals; /* Allocatd by caller */ + bool *read; + bool *written; + enum regmap_endian reg_endian; + bool (*noinc_reg)(struct regmap_ram_data *data, unsigned int reg); +}; + +/* + * Create a test register map with data stored in RAM, not intended + * for practical use. + */ +struct regmap *__regmap_init_ram(struct device *dev, + const struct regmap_config *config, + struct regmap_ram_data *data, + struct lock_class_key *lock_key, + const char *lock_name); + +#define regmap_init_ram(dev, config, data) \ + __regmap_lockdep_wrapper(__regmap_init_ram, #dev, dev, config, data) + +struct regmap *__regmap_init_raw_ram(struct device *dev, + const struct regmap_config *config, + struct regmap_ram_data *data, + struct lock_class_key *lock_key, + const char *lock_name); + +#define regmap_init_raw_ram(dev, config, data) \ + __regmap_lockdep_wrapper(__regmap_init_raw_ram, #dev, dev, config, data) + #endif diff --git a/drivers/base/regmap/regcache-flat.c b/drivers/base/regmap/regcache-flat.c index d9762e41959b..53cc59c84e2f 100644 --- a/drivers/base/regmap/regcache-flat.c +++ b/drivers/base/regmap/regcache-flat.c @@ -1,53 +1,113 @@ -/* - * Register cache access API - flat caching support - * - * Copyright 2012 Wolfson Microelectronics plc - * - * Author: Mark Brown <broonie@opensource.wolfsonmicro.com> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - */ - -#include <linux/slab.h> +// SPDX-License-Identifier: GPL-2.0 +// +// Register cache access API - flat caching support +// +// Copyright 2012 Wolfson Microelectronics plc +// +// Author: Mark Brown <broonie@opensource.wolfsonmicro.com> + +#include <linux/bitmap.h> +#include <linux/bitops.h> #include <linux/device.h> +#include <linux/limits.h> +#include <linux/overflow.h> #include <linux/seq_file.h> +#include <linux/slab.h> #include "internal.h" +static inline unsigned int regcache_flat_get_index(const struct regmap *map, + unsigned int reg) +{ + return regcache_get_index_by_order(map, reg); +} + +struct regcache_flat_data { + unsigned long *valid; + unsigned int data[]; +}; + static int regcache_flat_init(struct regmap *map) { - int i; - unsigned int *cache; + unsigned int cache_size; + struct regcache_flat_data *cache; - map->cache = kzalloc(sizeof(unsigned int) * (map->max_register + 1), - GFP_KERNEL); - if (!map->cache) + if (!map || map->reg_stride_order < 0 || !map->max_register_is_set) + return -EINVAL; + + cache_size = regcache_flat_get_index(map, map->max_register) + 1; + cache = kzalloc(struct_size(cache, data, cache_size), map->alloc_flags); + if (!cache) return -ENOMEM; - cache = map->cache; + cache->valid = bitmap_zalloc(cache_size, map->alloc_flags); + if (!cache->valid) + goto err_free; - for (i = 0; i < map->num_reg_defaults; i++) - cache[map->reg_defaults[i].reg] = map->reg_defaults[i].def; + map->cache = cache; return 0; + +err_free: + kfree(cache); + return -ENOMEM; } static int regcache_flat_exit(struct regmap *map) { - kfree(map->cache); + struct regcache_flat_data *cache = map->cache; + + if (cache) + bitmap_free(cache->valid); + + kfree(cache); map->cache = NULL; return 0; } +static int regcache_flat_populate(struct regmap *map) +{ + struct regcache_flat_data *cache = map->cache; + unsigned int i; + + for (i = 0; i < map->num_reg_defaults; i++) { + unsigned int reg = map->reg_defaults[i].reg; + unsigned int index = regcache_flat_get_index(map, reg); + + cache->data[index] = map->reg_defaults[i].def; + __set_bit(index, cache->valid); + } + + return 0; +} + static int regcache_flat_read(struct regmap *map, unsigned int reg, unsigned int *value) { - unsigned int *cache = map->cache; + struct regcache_flat_data *cache = map->cache; + unsigned int index = regcache_flat_get_index(map, reg); - *value = cache[reg]; + /* legacy behavior: ignore validity, but warn the user */ + if (unlikely(!test_bit(index, cache->valid))) + dev_warn_once(map->dev, + "using zero-initialized flat cache, this may cause unexpected behavior"); + + *value = cache->data[index]; + + return 0; +} + +static int regcache_flat_sparse_read(struct regmap *map, + unsigned int reg, unsigned int *value) +{ + struct regcache_flat_data *cache = map->cache; + unsigned int index = regcache_flat_get_index(map, reg); + + if (unlikely(!test_bit(index, cache->valid))) + return -ENOENT; + + *value = cache->data[index]; return 0; } @@ -55,9 +115,23 @@ static int regcache_flat_read(struct regmap *map, static int regcache_flat_write(struct regmap *map, unsigned int reg, unsigned int value) { - unsigned int *cache = map->cache; + struct regcache_flat_data *cache = map->cache; + unsigned int index = regcache_flat_get_index(map, reg); - cache[reg] = value; + cache->data[index] = value; + __set_bit(index, cache->valid); + + return 0; +} + +static int regcache_flat_drop(struct regmap *map, unsigned int min, + unsigned int max) +{ + struct regcache_flat_data *cache = map->cache; + unsigned int bitmap_min = regcache_flat_get_index(map, min); + unsigned int bitmap_max = regcache_flat_get_index(map, max); + + bitmap_clear(cache->valid, bitmap_min, bitmap_max + 1 - bitmap_min); return 0; } @@ -67,6 +141,18 @@ struct regcache_ops regcache_flat_ops = { .name = "flat", .init = regcache_flat_init, .exit = regcache_flat_exit, + .populate = regcache_flat_populate, .read = regcache_flat_read, .write = regcache_flat_write, }; + +struct regcache_ops regcache_flat_sparse_ops = { + .type = REGCACHE_FLAT_S, + .name = "flat-sparse", + .init = regcache_flat_init, + .exit = regcache_flat_exit, + .populate = regcache_flat_populate, + .read = regcache_flat_sparse_read, + .write = regcache_flat_write, + .drop = regcache_flat_drop, +}; diff --git a/drivers/base/regmap/regcache-lzo.c b/drivers/base/regmap/regcache-lzo.c deleted file mode 100644 index e210a6d1406a..000000000000 --- a/drivers/base/regmap/regcache-lzo.c +++ /dev/null @@ -1,378 +0,0 @@ -/* - * Register cache access API - LZO caching support - * - * Copyright 2011 Wolfson Microelectronics plc - * - * Author: Dimitris Papastamos <dp@opensource.wolfsonmicro.com> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - */ - -#include <linux/slab.h> -#include <linux/device.h> -#include <linux/lzo.h> - -#include "internal.h" - -static int regcache_lzo_exit(struct regmap *map); - -struct regcache_lzo_ctx { - void *wmem; - void *dst; - const void *src; - size_t src_len; - size_t dst_len; - size_t decompressed_size; - unsigned long *sync_bmp; - int sync_bmp_nbits; -}; - -#define LZO_BLOCK_NUM 8 -static int regcache_lzo_block_count(struct regmap *map) -{ - return LZO_BLOCK_NUM; -} - -static int regcache_lzo_prepare(struct regcache_lzo_ctx *lzo_ctx) -{ - lzo_ctx->wmem = kmalloc(LZO1X_MEM_COMPRESS, GFP_KERNEL); - if (!lzo_ctx->wmem) - return -ENOMEM; - return 0; -} - -static int regcache_lzo_compress(struct regcache_lzo_ctx *lzo_ctx) -{ - size_t compress_size; - int ret; - - ret = lzo1x_1_compress(lzo_ctx->src, lzo_ctx->src_len, - lzo_ctx->dst, &compress_size, lzo_ctx->wmem); - if (ret != LZO_E_OK || compress_size > lzo_ctx->dst_len) - return -EINVAL; - lzo_ctx->dst_len = compress_size; - return 0; -} - -static int regcache_lzo_decompress(struct regcache_lzo_ctx *lzo_ctx) -{ - size_t dst_len; - int ret; - - dst_len = lzo_ctx->dst_len; - ret = lzo1x_decompress_safe(lzo_ctx->src, lzo_ctx->src_len, - lzo_ctx->dst, &dst_len); - if (ret != LZO_E_OK || dst_len != lzo_ctx->dst_len) - return -EINVAL; - return 0; -} - -static int regcache_lzo_compress_cache_block(struct regmap *map, - struct regcache_lzo_ctx *lzo_ctx) -{ - int ret; - - lzo_ctx->dst_len = lzo1x_worst_compress(PAGE_SIZE); - lzo_ctx->dst = kmalloc(lzo_ctx->dst_len, GFP_KERNEL); - if (!lzo_ctx->dst) { - lzo_ctx->dst_len = 0; - return -ENOMEM; - } - - ret = regcache_lzo_compress(lzo_ctx); - if (ret < 0) - return ret; - return 0; -} - -static int regcache_lzo_decompress_cache_block(struct regmap *map, - struct regcache_lzo_ctx *lzo_ctx) -{ - int ret; - - lzo_ctx->dst_len = lzo_ctx->decompressed_size; - lzo_ctx->dst = kmalloc(lzo_ctx->dst_len, GFP_KERNEL); - if (!lzo_ctx->dst) { - lzo_ctx->dst_len = 0; - return -ENOMEM; - } - - ret = regcache_lzo_decompress(lzo_ctx); - if (ret < 0) - return ret; - return 0; -} - -static inline int regcache_lzo_get_blkindex(struct regmap *map, - unsigned int reg) -{ - return ((reg / map->reg_stride) * map->cache_word_size) / - DIV_ROUND_UP(map->cache_size_raw, - regcache_lzo_block_count(map)); -} - -static inline int regcache_lzo_get_blkpos(struct regmap *map, - unsigned int reg) -{ - return (reg / map->reg_stride) % - (DIV_ROUND_UP(map->cache_size_raw, - regcache_lzo_block_count(map)) / - map->cache_word_size); -} - -static inline int regcache_lzo_get_blksize(struct regmap *map) -{ - return DIV_ROUND_UP(map->cache_size_raw, - regcache_lzo_block_count(map)); -} - -static int regcache_lzo_init(struct regmap *map) -{ - struct regcache_lzo_ctx **lzo_blocks; - size_t bmp_size; - int ret, i, blksize, blkcount; - const char *p, *end; - unsigned long *sync_bmp; - - ret = 0; - - blkcount = regcache_lzo_block_count(map); - map->cache = kzalloc(blkcount * sizeof *lzo_blocks, - GFP_KERNEL); - if (!map->cache) - return -ENOMEM; - lzo_blocks = map->cache; - - /* - * allocate a bitmap to be used when syncing the cache with - * the hardware. Each time a register is modified, the corresponding - * bit is set in the bitmap, so we know that we have to sync - * that register. - */ - bmp_size = map->num_reg_defaults_raw; - sync_bmp = kmalloc(BITS_TO_LONGS(bmp_size) * sizeof(long), - GFP_KERNEL); - if (!sync_bmp) { - ret = -ENOMEM; - goto err; - } - bitmap_zero(sync_bmp, bmp_size); - - /* allocate the lzo blocks and initialize them */ - for (i = 0; i < blkcount; i++) { - lzo_blocks[i] = kzalloc(sizeof **lzo_blocks, - GFP_KERNEL); - if (!lzo_blocks[i]) { - kfree(sync_bmp); - ret = -ENOMEM; - goto err; - } - lzo_blocks[i]->sync_bmp = sync_bmp; - lzo_blocks[i]->sync_bmp_nbits = bmp_size; - /* alloc the working space for the compressed block */ - ret = regcache_lzo_prepare(lzo_blocks[i]); - if (ret < 0) - goto err; - } - - blksize = regcache_lzo_get_blksize(map); - p = map->reg_defaults_raw; - end = map->reg_defaults_raw + map->cache_size_raw; - /* compress the register map and fill the lzo blocks */ - for (i = 0; i < blkcount; i++, p += blksize) { - lzo_blocks[i]->src = p; - if (p + blksize > end) - lzo_blocks[i]->src_len = end - p; - else - lzo_blocks[i]->src_len = blksize; - ret = regcache_lzo_compress_cache_block(map, - lzo_blocks[i]); - if (ret < 0) - goto err; - lzo_blocks[i]->decompressed_size = - lzo_blocks[i]->src_len; - } - - return 0; -err: - regcache_lzo_exit(map); - return ret; -} - -static int regcache_lzo_exit(struct regmap *map) -{ - struct regcache_lzo_ctx **lzo_blocks; - int i, blkcount; - - lzo_blocks = map->cache; - if (!lzo_blocks) - return 0; - - blkcount = regcache_lzo_block_count(map); - /* - * the pointer to the bitmap used for syncing the cache - * is shared amongst all lzo_blocks. Ensure it is freed - * only once. - */ - if (lzo_blocks[0]) - kfree(lzo_blocks[0]->sync_bmp); - for (i = 0; i < blkcount; i++) { - if (lzo_blocks[i]) { - kfree(lzo_blocks[i]->wmem); - kfree(lzo_blocks[i]->dst); - } - /* each lzo_block is a pointer returned by kmalloc or NULL */ - kfree(lzo_blocks[i]); - } - kfree(lzo_blocks); - map->cache = NULL; - return 0; -} - -static int regcache_lzo_read(struct regmap *map, - unsigned int reg, unsigned int *value) -{ - struct regcache_lzo_ctx *lzo_block, **lzo_blocks; - int ret, blkindex, blkpos; - size_t blksize, tmp_dst_len; - void *tmp_dst; - - /* index of the compressed lzo block */ - blkindex = regcache_lzo_get_blkindex(map, reg); - /* register index within the decompressed block */ - blkpos = regcache_lzo_get_blkpos(map, reg); - /* size of the compressed block */ - blksize = regcache_lzo_get_blksize(map); - lzo_blocks = map->cache; - lzo_block = lzo_blocks[blkindex]; - - /* save the pointer and length of the compressed block */ - tmp_dst = lzo_block->dst; - tmp_dst_len = lzo_block->dst_len; - - /* prepare the source to be the compressed block */ - lzo_block->src = lzo_block->dst; - lzo_block->src_len = lzo_block->dst_len; - - /* decompress the block */ - ret = regcache_lzo_decompress_cache_block(map, lzo_block); - if (ret >= 0) - /* fetch the value from the cache */ - *value = regcache_get_val(map, lzo_block->dst, blkpos); - - kfree(lzo_block->dst); - /* restore the pointer and length of the compressed block */ - lzo_block->dst = tmp_dst; - lzo_block->dst_len = tmp_dst_len; - - return ret; -} - -static int regcache_lzo_write(struct regmap *map, - unsigned int reg, unsigned int value) -{ - struct regcache_lzo_ctx *lzo_block, **lzo_blocks; - int ret, blkindex, blkpos; - size_t blksize, tmp_dst_len; - void *tmp_dst; - - /* index of the compressed lzo block */ - blkindex = regcache_lzo_get_blkindex(map, reg); - /* register index within the decompressed block */ - blkpos = regcache_lzo_get_blkpos(map, reg); - /* size of the compressed block */ - blksize = regcache_lzo_get_blksize(map); - lzo_blocks = map->cache; - lzo_block = lzo_blocks[blkindex]; - - /* save the pointer and length of the compressed block */ - tmp_dst = lzo_block->dst; - tmp_dst_len = lzo_block->dst_len; - - /* prepare the source to be the compressed block */ - lzo_block->src = lzo_block->dst; - lzo_block->src_len = lzo_block->dst_len; - - /* decompress the block */ - ret = regcache_lzo_decompress_cache_block(map, lzo_block); - if (ret < 0) { - kfree(lzo_block->dst); - goto out; - } - - /* write the new value to the cache */ - if (regcache_set_val(map, lzo_block->dst, blkpos, value)) { - kfree(lzo_block->dst); - goto out; - } - - /* prepare the source to be the decompressed block */ - lzo_block->src = lzo_block->dst; - lzo_block->src_len = lzo_block->dst_len; - - /* compress the block */ - ret = regcache_lzo_compress_cache_block(map, lzo_block); - if (ret < 0) { - kfree(lzo_block->dst); - kfree(lzo_block->src); - goto out; - } - - /* set the bit so we know we have to sync this register */ - set_bit(reg / map->reg_stride, lzo_block->sync_bmp); - kfree(tmp_dst); - kfree(lzo_block->src); - return 0; -out: - lzo_block->dst = tmp_dst; - lzo_block->dst_len = tmp_dst_len; - return ret; -} - -static int regcache_lzo_sync(struct regmap *map, unsigned int min, - unsigned int max) -{ - struct regcache_lzo_ctx **lzo_blocks; - unsigned int val; - int i; - int ret; - - lzo_blocks = map->cache; - i = min; - for_each_set_bit_from(i, lzo_blocks[0]->sync_bmp, - lzo_blocks[0]->sync_bmp_nbits) { - if (i > max) - continue; - - ret = regcache_read(map, i, &val); - if (ret) - return ret; - - /* Is this the hardware default? If so skip. */ - ret = regcache_lookup_reg(map, i); - if (ret > 0 && val == map->reg_defaults[ret].def) - continue; - - map->cache_bypass = 1; - ret = _regmap_write(map, i, val); - map->cache_bypass = 0; - if (ret) - return ret; - dev_dbg(map->dev, "Synced register %#x, value %#x\n", - i, val); - } - - return 0; -} - -struct regcache_ops regcache_lzo_ops = { - .type = REGCACHE_COMPRESSED, - .name = "lzo", - .init = regcache_lzo_init, - .exit = regcache_lzo_exit, - .read = regcache_lzo_read, - .write = regcache_lzo_write, - .sync = regcache_lzo_sync -}; diff --git a/drivers/base/regmap/regcache-maple.c b/drivers/base/regmap/regcache-maple.c new file mode 100644 index 000000000000..ca1c72b68f31 --- /dev/null +++ b/drivers/base/regmap/regcache-maple.c @@ -0,0 +1,395 @@ +// SPDX-License-Identifier: GPL-2.0 +// +// Register cache access API - maple tree based cache +// +// Copyright 2023 Arm, Ltd +// +// Author: Mark Brown <broonie@kernel.org> + +#include <linux/debugfs.h> +#include <linux/device.h> +#include <linux/maple_tree.h> +#include <linux/slab.h> + +#include "internal.h" + +static int regcache_maple_read(struct regmap *map, + unsigned int reg, unsigned int *value) +{ + struct maple_tree *mt = map->cache; + MA_STATE(mas, mt, reg, reg); + unsigned long *entry; + + rcu_read_lock(); + + entry = mas_walk(&mas); + if (!entry) { + rcu_read_unlock(); + return -ENOENT; + } + + *value = entry[reg - mas.index]; + + rcu_read_unlock(); + + return 0; +} + +static int regcache_maple_write(struct regmap *map, unsigned int reg, + unsigned int val) +{ + struct maple_tree *mt = map->cache; + MA_STATE(mas, mt, reg, reg); + unsigned long *entry, *upper, *lower; + unsigned long index, last; + size_t lower_sz, upper_sz; + int ret; + + rcu_read_lock(); + + entry = mas_walk(&mas); + if (entry) { + entry[reg - mas.index] = val; + rcu_read_unlock(); + return 0; + } + + /* Any adjacent entries to extend/merge? */ + mas_set_range(&mas, reg - 1, reg + 1); + index = reg; + last = reg; + + lower = mas_find(&mas, reg - 1); + if (lower) { + index = mas.index; + lower_sz = (mas.last - mas.index + 1) * sizeof(unsigned long); + } + + upper = mas_find(&mas, reg + 1); + if (upper) { + last = mas.last; + upper_sz = (mas.last - mas.index + 1) * sizeof(unsigned long); + } + + rcu_read_unlock(); + + entry = kmalloc_array(last - index + 1, sizeof(*entry), map->alloc_flags); + if (!entry) + return -ENOMEM; + + if (lower) + memcpy(entry, lower, lower_sz); + entry[reg - index] = val; + if (upper) + memcpy(&entry[reg - index + 1], upper, upper_sz); + + /* + * This is safe because the regmap lock means the Maple lock + * is redundant, but we need to take it due to lockdep asserts + * in the maple tree code. + */ + mas_lock(&mas); + + mas_set_range(&mas, index, last); + ret = mas_store_gfp(&mas, entry, map->alloc_flags); + + mas_unlock(&mas); + + if (ret == 0) { + kfree(lower); + kfree(upper); + } + + return ret; +} + +static int regcache_maple_drop(struct regmap *map, unsigned int min, + unsigned int max) +{ + struct maple_tree *mt = map->cache; + MA_STATE(mas, mt, min, max); + unsigned long *entry, *lower, *upper; + /* initialized to work around false-positive -Wuninitialized warning */ + unsigned long lower_index = 0, lower_last = 0; + unsigned long upper_index, upper_last; + int ret = 0; + + lower = NULL; + upper = NULL; + + mas_lock(&mas); + + mas_for_each(&mas, entry, max) { + /* + * This is safe because the regmap lock means the + * Maple lock is redundant, but we need to take it due + * to lockdep asserts in the maple tree code. + */ + mas_unlock(&mas); + + /* Do we need to save any of this entry? */ + if (mas.index < min) { + lower_index = mas.index; + lower_last = min -1; + + lower = kmemdup_array(entry, + min - mas.index, sizeof(*lower), + map->alloc_flags); + if (!lower) { + ret = -ENOMEM; + goto out_unlocked; + } + } + + if (mas.last > max) { + upper_index = max + 1; + upper_last = mas.last; + + upper = kmemdup_array(&entry[max - mas.index + 1], + mas.last - max, sizeof(*upper), + map->alloc_flags); + if (!upper) { + ret = -ENOMEM; + goto out_unlocked; + } + } + + kfree(entry); + mas_lock(&mas); + mas_erase(&mas); + + /* Insert new nodes with the saved data */ + if (lower) { + mas_set_range(&mas, lower_index, lower_last); + ret = mas_store_gfp(&mas, lower, map->alloc_flags); + if (ret != 0) + goto out; + lower = NULL; + } + + if (upper) { + mas_set_range(&mas, upper_index, upper_last); + ret = mas_store_gfp(&mas, upper, map->alloc_flags); + if (ret != 0) + goto out; + upper = NULL; + } + } + +out: + mas_unlock(&mas); +out_unlocked: + kfree(lower); + kfree(upper); + + return ret; +} + +static int regcache_maple_sync_block(struct regmap *map, unsigned long *entry, + struct ma_state *mas, + unsigned int min, unsigned int max) +{ + void *buf; + unsigned long r; + size_t val_bytes = map->format.val_bytes; + int ret = 0; + + mas_pause(mas); + rcu_read_unlock(); + + /* + * Use a raw write if writing more than one register to a + * device that supports raw writes to reduce transaction + * overheads. + */ + if (max - min > 1 && regmap_can_raw_write(map)) { + buf = kmalloc_array(max - min, val_bytes, map->alloc_flags); + if (!buf) { + ret = -ENOMEM; + goto out; + } + + /* Render the data for a raw write */ + for (r = min; r < max; r++) { + regcache_set_val(map, buf, r - min, + entry[r - mas->index]); + } + + ret = _regmap_raw_write(map, min, buf, (max - min) * val_bytes, + false); + + kfree(buf); + } else { + for (r = min; r < max; r++) { + ret = _regmap_write(map, r, + entry[r - mas->index]); + if (ret != 0) + goto out; + } + } + +out: + rcu_read_lock(); + + return ret; +} + +static int regcache_maple_sync(struct regmap *map, unsigned int min, + unsigned int max) +{ + struct maple_tree *mt = map->cache; + unsigned long *entry; + MA_STATE(mas, mt, min, max); + unsigned long lmin = min; + unsigned long lmax = max; + unsigned int r, v, sync_start; + int ret = 0; + bool sync_needed = false; + + map->cache_bypass = true; + + rcu_read_lock(); + + mas_for_each(&mas, entry, max) { + for (r = max(mas.index, lmin); r <= min(mas.last, lmax); r++) { + v = entry[r - mas.index]; + + if (regcache_reg_needs_sync(map, r, v)) { + if (!sync_needed) { + sync_start = r; + sync_needed = true; + } + continue; + } + + if (!sync_needed) + continue; + + ret = regcache_maple_sync_block(map, entry, &mas, + sync_start, r); + if (ret != 0) + goto out; + sync_needed = false; + } + + if (sync_needed) { + ret = regcache_maple_sync_block(map, entry, &mas, + sync_start, r); + if (ret != 0) + goto out; + sync_needed = false; + } + } + +out: + rcu_read_unlock(); + + map->cache_bypass = false; + + return ret; +} + +static int regcache_maple_init(struct regmap *map) +{ + struct maple_tree *mt; + + mt = kmalloc(sizeof(*mt), map->alloc_flags); + if (!mt) + return -ENOMEM; + map->cache = mt; + + mt_init(mt); + + if (!mt_external_lock(mt) && map->lock_key) + lockdep_set_class_and_subclass(&mt->ma_lock, map->lock_key, 1); + + return 0; +} + +static int regcache_maple_exit(struct regmap *map) +{ + struct maple_tree *mt = map->cache; + MA_STATE(mas, mt, 0, UINT_MAX); + unsigned int *entry; + + /* if we've already been called then just return */ + if (!mt) + return 0; + + mas_lock(&mas); + mas_for_each(&mas, entry, UINT_MAX) + kfree(entry); + __mt_destroy(mt); + mas_unlock(&mas); + + kfree(mt); + map->cache = NULL; + + return 0; +} + +static int regcache_maple_insert_block(struct regmap *map, int first, + int last) +{ + struct maple_tree *mt = map->cache; + MA_STATE(mas, mt, first, last); + unsigned long *entry; + int i, ret; + + entry = kmalloc_array(last - first + 1, sizeof(*entry), map->alloc_flags); + if (!entry) + return -ENOMEM; + + for (i = 0; i < last - first + 1; i++) + entry[i] = map->reg_defaults[first + i].def; + + mas_lock(&mas); + + mas_set_range(&mas, map->reg_defaults[first].reg, + map->reg_defaults[last].reg); + ret = mas_store_gfp(&mas, entry, map->alloc_flags); + + mas_unlock(&mas); + + if (ret) + kfree(entry); + + return ret; +} + +static int regcache_maple_populate(struct regmap *map) +{ + int i; + int ret; + int range_start; + + range_start = 0; + + /* Scan for ranges of contiguous registers */ + for (i = 1; i < map->num_reg_defaults; i++) { + if (map->reg_defaults[i].reg != + map->reg_defaults[i - 1].reg + 1) { + ret = regcache_maple_insert_block(map, range_start, + i - 1); + if (ret != 0) + return ret; + + range_start = i; + } + } + + /* Add the last block */ + return regcache_maple_insert_block(map, range_start, map->num_reg_defaults - 1); +} + +struct regcache_ops regcache_maple_ops = { + .type = REGCACHE_MAPLE, + .name = "maple", + .init = regcache_maple_init, + .exit = regcache_maple_exit, + .populate = regcache_maple_populate, + .read = regcache_maple_read, + .write = regcache_maple_write, + .drop = regcache_maple_drop, + .sync = regcache_maple_sync, +}; diff --git a/drivers/base/regmap/regcache-rbtree.c b/drivers/base/regmap/regcache-rbtree.c index 5c1435c4e210..3344b82c3799 100644 --- a/drivers/base/regmap/regcache-rbtree.c +++ b/drivers/base/regmap/regcache-rbtree.c @@ -1,20 +1,16 @@ -/* - * Register cache access API - rbtree caching support - * - * Copyright 2011 Wolfson Microelectronics plc - * - * Author: Dimitris Papastamos <dp@opensource.wolfsonmicro.com> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - */ +// SPDX-License-Identifier: GPL-2.0 +// +// Register cache access API - rbtree caching support +// +// Copyright 2011 Wolfson Microelectronics plc +// +// Author: Dimitris Papastamos <dp@opensource.wolfsonmicro.com> -#include <linux/slab.h> -#include <linux/device.h> #include <linux/debugfs.h> +#include <linux/device.h> #include <linux/rbtree.h> #include <linux/seq_file.h> +#include <linux/slab.h> #include "internal.h" @@ -23,15 +19,17 @@ static int regcache_rbtree_write(struct regmap *map, unsigned int reg, static int regcache_rbtree_exit(struct regmap *map); struct regcache_rbtree_node { - /* the actual rbtree node holding this block */ - struct rb_node node; - /* base register handled by this block */ - unsigned int base_reg; /* block of adjacent registers */ void *block; + /* Which registers are present */ + unsigned long *cache_present; + /* base register handled by this block */ + unsigned int base_reg; /* number of registers available in the block */ unsigned int blklen; -} __attribute__ ((packed)); + /* the actual rbtree node holding this block */ + struct rb_node node; +}; struct regcache_rbtree_ctx { struct rb_root root; @@ -57,6 +55,7 @@ static void regcache_rbtree_set_register(struct regmap *map, struct regcache_rbtree_node *rbnode, unsigned int idx, unsigned int val) { + set_bit(idx, rbnode->cache_present); regcache_set_val(map, rbnode->block, idx, val); } @@ -78,7 +77,7 @@ static struct regcache_rbtree_node *regcache_rbtree_lookup(struct regmap *map, node = rbtree_ctx->root.rb_node; while (node) { - rbnode = container_of(node, struct regcache_rbtree_node, node); + rbnode = rb_entry(node, struct regcache_rbtree_node, node); regcache_rbtree_get_base_top_reg(map, rbnode, &base_reg, &top_reg); if (reg >= base_reg && reg <= top_reg) { @@ -105,8 +104,7 @@ static int regcache_rbtree_insert(struct regmap *map, struct rb_root *root, parent = NULL; new = &root->rb_node; while (*new) { - rbnode_tmp = container_of(*new, struct regcache_rbtree_node, - node); + rbnode_tmp = rb_entry(*new, struct regcache_rbtree_node, node); /* base and top registers of the current rbnode */ regcache_rbtree_get_base_top_reg(map, rbnode_tmp, &base_reg_tmp, &top_reg_tmp); @@ -146,13 +144,13 @@ static int rbtree_show(struct seq_file *s, void *ignored) map->lock(map->lock_arg); mem_size = sizeof(*rbtree_ctx); - mem_size += BITS_TO_LONGS(map->cache_present_nbits) * sizeof(long); for (node = rb_first(&rbtree_ctx->root); node != NULL; node = rb_next(node)) { - n = container_of(node, struct regcache_rbtree_node, node); + n = rb_entry(node, struct regcache_rbtree_node, node); mem_size += sizeof(*n); mem_size += (n->blklen * map->cache_word_size); + mem_size += BITS_TO_LONGS(n->blklen) * sizeof(long); regcache_rbtree_get_base_top_reg(map, n, &base, &top); this_registers = ((top - base) / map->reg_stride) + 1; @@ -175,35 +173,19 @@ static int rbtree_show(struct seq_file *s, void *ignored) return 0; } -static int rbtree_open(struct inode *inode, struct file *file) -{ - return single_open(file, rbtree_show, inode->i_private); -} - -static const struct file_operations rbtree_fops = { - .open = rbtree_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; +DEFINE_SHOW_ATTRIBUTE(rbtree); static void rbtree_debugfs_init(struct regmap *map) { debugfs_create_file("rbtree", 0400, map->debugfs, map, &rbtree_fops); } -#else -static void rbtree_debugfs_init(struct regmap *map) -{ -} #endif static int regcache_rbtree_init(struct regmap *map) { struct regcache_rbtree_ctx *rbtree_ctx; - int i; - int ret; - map->cache = kmalloc(sizeof *rbtree_ctx, GFP_KERNEL); + map->cache = kmalloc(sizeof *rbtree_ctx, map->alloc_flags); if (!map->cache) return -ENOMEM; @@ -211,21 +193,7 @@ static int regcache_rbtree_init(struct regmap *map) rbtree_ctx->root = RB_ROOT; rbtree_ctx->cached_rbnode = NULL; - for (i = 0; i < map->num_reg_defaults; i++) { - ret = regcache_rbtree_write(map, - map->reg_defaults[i].reg, - map->reg_defaults[i].def); - if (ret) - goto err; - } - - rbtree_debugfs_init(map); - return 0; - -err: - regcache_rbtree_exit(map); - return ret; } static int regcache_rbtree_exit(struct regmap *map) @@ -245,6 +213,7 @@ static int regcache_rbtree_exit(struct regmap *map) rbtree_node = rb_entry(next, struct regcache_rbtree_node, node); next = rb_next(&rbtree_node->node); rb_erase(&rbtree_node->node, &rbtree_ctx->root); + kfree(rbtree_node->cache_present); kfree(rbtree_node->block); kfree(rbtree_node); } @@ -256,6 +225,22 @@ static int regcache_rbtree_exit(struct regmap *map) return 0; } +static int regcache_rbtree_populate(struct regmap *map) +{ + unsigned int i; + int ret; + + for (i = 0; i < map->num_reg_defaults; i++) { + ret = regcache_rbtree_write(map, + map->reg_defaults[i].reg, + map->reg_defaults[i].def); + if (ret) + return ret; + } + + return 0; +} + static int regcache_rbtree_read(struct regmap *map, unsigned int reg, unsigned int *value) { @@ -265,7 +250,7 @@ static int regcache_rbtree_read(struct regmap *map, rbnode = regcache_rbtree_lookup(map, reg); if (rbnode) { reg_tmp = (reg - rbnode->base_reg) / map->reg_stride; - if (!regcache_reg_present(map, reg)) + if (!test_bit(reg_tmp, rbnode->cache_present)) return -ENOENT; *value = regcache_rbtree_get_register(map, rbnode, reg_tmp); } else { @@ -278,27 +263,51 @@ static int regcache_rbtree_read(struct regmap *map, static int regcache_rbtree_insert_to_block(struct regmap *map, struct regcache_rbtree_node *rbnode, - unsigned int pos, unsigned int reg, + unsigned int base_reg, + unsigned int top_reg, + unsigned int reg, unsigned int value) { + unsigned int blklen; + unsigned int pos, offset; + unsigned long *present; u8 *blk; - blk = krealloc(rbnode->block, - (rbnode->blklen + 1) * map->cache_word_size, - GFP_KERNEL); + blklen = (top_reg - base_reg) / map->reg_stride + 1; + pos = (reg - base_reg) / map->reg_stride; + offset = (rbnode->base_reg - base_reg) / map->reg_stride; + + blk = krealloc_array(rbnode->block, blklen, map->cache_word_size, map->alloc_flags); if (!blk) return -ENOMEM; + rbnode->block = blk; + + if (BITS_TO_LONGS(blklen) > BITS_TO_LONGS(rbnode->blklen)) { + present = krealloc_array(rbnode->cache_present, + BITS_TO_LONGS(blklen), sizeof(*present), + map->alloc_flags); + if (!present) + return -ENOMEM; + + memset(present + BITS_TO_LONGS(rbnode->blklen), 0, + (BITS_TO_LONGS(blklen) - BITS_TO_LONGS(rbnode->blklen)) + * sizeof(*present)); + } else { + present = rbnode->cache_present; + } + /* insert the register value in the correct place in the rbnode block */ - memmove(blk + (pos + 1) * map->cache_word_size, - blk + pos * map->cache_word_size, - (rbnode->blklen - pos) * map->cache_word_size); + if (pos == 0) { + memmove(blk + offset * map->cache_word_size, + blk, rbnode->blklen * map->cache_word_size); + bitmap_shift_left(present, present, offset, blklen); + } /* update the rbnode block, its size and the base register */ - rbnode->block = blk; - rbnode->blklen++; - if (!pos) - rbnode->base_reg = reg; + rbnode->blklen = blklen; + rbnode->base_reg = base_reg; + rbnode->cache_present = present; regcache_rbtree_set_register(map, rbnode, pos, value); return 0; @@ -311,7 +320,7 @@ regcache_rbtree_node_alloc(struct regmap *map, unsigned int reg) const struct regmap_range *range; int i; - rbnode = kzalloc(sizeof(*rbnode), GFP_KERNEL); + rbnode = kzalloc(sizeof(*rbnode), map->alloc_flags); if (!rbnode) return NULL; @@ -325,25 +334,35 @@ regcache_rbtree_node_alloc(struct regmap *map, unsigned int reg) if (i != map->rd_table->n_yes_ranges) { range = &map->rd_table->yes_ranges[i]; - rbnode->blklen = range->range_max - range->range_min - + 1; + rbnode->blklen = (range->range_max - range->range_min) / + map->reg_stride + 1; rbnode->base_reg = range->range_min; } } if (!rbnode->blklen) { - rbnode->blklen = sizeof(*rbnode); + rbnode->blklen = 1; rbnode->base_reg = reg; } - rbnode->block = kmalloc(rbnode->blklen * map->cache_word_size, - GFP_KERNEL); - if (!rbnode->block) { - kfree(rbnode); - return NULL; - } + rbnode->block = kmalloc_array(rbnode->blklen, map->cache_word_size, + map->alloc_flags); + if (!rbnode->block) + goto err_free; + + rbnode->cache_present = kcalloc(BITS_TO_LONGS(rbnode->blklen), + sizeof(*rbnode->cache_present), + map->alloc_flags); + if (!rbnode->cache_present) + goto err_free_block; return rbnode; + +err_free_block: + kfree(rbnode->block); +err_free: + kfree(rbnode); + return NULL; } static int regcache_rbtree_write(struct regmap *map, unsigned int reg, @@ -353,15 +372,9 @@ static int regcache_rbtree_write(struct regmap *map, unsigned int reg, struct regcache_rbtree_node *rbnode, *rbnode_tmp; struct rb_node *node; unsigned int reg_tmp; - unsigned int pos; - int i; int ret; rbtree_ctx = map->cache; - /* update the reg_present bitmap, make space if necessary */ - ret = regcache_set_reg_present(map, reg); - if (ret < 0) - return ret; /* if we can't locate it in the cached rbnode we'll have * to traverse the rbtree looking for it. @@ -371,30 +384,66 @@ static int regcache_rbtree_write(struct regmap *map, unsigned int reg, reg_tmp = (reg - rbnode->base_reg) / map->reg_stride; regcache_rbtree_set_register(map, rbnode, reg_tmp, value); } else { + unsigned int base_reg, top_reg; + unsigned int new_base_reg, new_top_reg; + unsigned int min, max; + unsigned int max_dist; + unsigned int dist, best_dist = UINT_MAX; + + max_dist = map->reg_stride * sizeof(*rbnode_tmp) / + map->cache_word_size; + if (reg < max_dist) + min = 0; + else + min = reg - max_dist; + max = reg + max_dist; + /* look for an adjacent register to the one we are about to add */ - for (node = rb_first(&rbtree_ctx->root); node; - node = rb_next(node)) { + node = rbtree_ctx->root.rb_node; + while (node) { rbnode_tmp = rb_entry(node, struct regcache_rbtree_node, node); - for (i = 0; i < rbnode_tmp->blklen; i++) { - reg_tmp = rbnode_tmp->base_reg + - (i * map->reg_stride); - if (abs(reg_tmp - reg) != map->reg_stride) - continue; - /* decide where in the block to place our register */ - if (reg_tmp + map->reg_stride == reg) - pos = i + 1; + + regcache_rbtree_get_base_top_reg(map, rbnode_tmp, + &base_reg, &top_reg); + + if (base_reg <= max && top_reg >= min) { + if (reg < base_reg) + dist = base_reg - reg; + else if (reg > top_reg) + dist = reg - top_reg; else - pos = i; - ret = regcache_rbtree_insert_to_block(map, - rbnode_tmp, - pos, reg, - value); - if (ret) - return ret; - rbtree_ctx->cached_rbnode = rbnode_tmp; - return 0; + dist = 0; + if (dist < best_dist) { + rbnode = rbnode_tmp; + best_dist = dist; + new_base_reg = min(reg, base_reg); + new_top_reg = max(reg, top_reg); + } } + + /* + * Keep looking, we want to choose the closest block, + * otherwise we might end up creating overlapping + * blocks, which breaks the rbtree. + */ + if (reg < base_reg) + node = node->rb_left; + else if (reg > top_reg) + node = node->rb_right; + else + break; + } + + if (rbnode) { + ret = regcache_rbtree_insert_to_block(map, rbnode, + new_base_reg, + new_top_reg, reg, + value); + if (ret) + return ret; + rbtree_ctx->cached_rbnode = rbnode; + return 0; } /* We did not manage to find a place to insert it in @@ -404,7 +453,8 @@ static int regcache_rbtree_write(struct regmap *map, unsigned int reg, if (!rbnode) return -ENOMEM; regcache_rbtree_set_register(map, rbnode, - reg - rbnode->base_reg, value); + (reg - rbnode->base_reg) / map->reg_stride, + value); regcache_rbtree_insert(map, &rbtree_ctx->root, rbnode); rbtree_ctx->cached_rbnode = rbnode; } @@ -418,43 +468,92 @@ static int regcache_rbtree_sync(struct regmap *map, unsigned int min, struct regcache_rbtree_ctx *rbtree_ctx; struct rb_node *node; struct regcache_rbtree_node *rbnode; + unsigned int base_reg, top_reg; + unsigned int start, end; int ret; - int base, end; + + map->async = true; rbtree_ctx = map->cache; for (node = rb_first(&rbtree_ctx->root); node; node = rb_next(node)) { rbnode = rb_entry(node, struct regcache_rbtree_node, node); - if (rbnode->base_reg > max) + regcache_rbtree_get_base_top_reg(map, rbnode, &base_reg, + &top_reg); + if (base_reg > max) break; - if (rbnode->base_reg + rbnode->blklen < min) + if (top_reg < min) continue; - if (min > rbnode->base_reg) - base = min - rbnode->base_reg; + if (min > base_reg) + start = (min - base_reg) / map->reg_stride; else - base = 0; + start = 0; - if (max < rbnode->base_reg + rbnode->blklen) - end = max - rbnode->base_reg + 1; + if (max < top_reg) + end = (max - base_reg) / map->reg_stride + 1; else end = rbnode->blklen; - ret = regcache_sync_block(map, rbnode->block, rbnode->base_reg, - base, end); + ret = regcache_sync_block(map, rbnode->block, + rbnode->cache_present, + rbnode->base_reg, start, end); if (ret != 0) return ret; } + map->async = false; + return regmap_async_complete(map); } +static int regcache_rbtree_drop(struct regmap *map, unsigned int min, + unsigned int max) +{ + struct regcache_rbtree_ctx *rbtree_ctx; + struct regcache_rbtree_node *rbnode; + struct rb_node *node; + unsigned int base_reg, top_reg; + unsigned int start, end; + + rbtree_ctx = map->cache; + for (node = rb_first(&rbtree_ctx->root); node; node = rb_next(node)) { + rbnode = rb_entry(node, struct regcache_rbtree_node, node); + + regcache_rbtree_get_base_top_reg(map, rbnode, &base_reg, + &top_reg); + if (base_reg > max) + break; + if (top_reg < min) + continue; + + if (min > base_reg) + start = (min - base_reg) / map->reg_stride; + else + start = 0; + + if (max < top_reg) + end = (max - base_reg) / map->reg_stride + 1; + else + end = rbnode->blklen; + + bitmap_clear(rbnode->cache_present, start, end - start); + } + + return 0; +} + struct regcache_ops regcache_rbtree_ops = { .type = REGCACHE_RBTREE, .name = "rbtree", .init = regcache_rbtree_init, .exit = regcache_rbtree_exit, + .populate = regcache_rbtree_populate, +#ifdef CONFIG_DEBUG_FS + .debugfs_init = rbtree_debugfs_init, +#endif .read = regcache_rbtree_read, .write = regcache_rbtree_write, - .sync = regcache_rbtree_sync + .sync = regcache_rbtree_sync, + .drop = regcache_rbtree_drop, }; diff --git a/drivers/base/regmap/regcache.c b/drivers/base/regmap/regcache.c index e69102696533..319c342bf5a0 100644 --- a/drivers/base/regmap/regcache.c +++ b/drivers/base/regmap/regcache.c @@ -1,83 +1,124 @@ -/* - * Register cache access API - * - * Copyright 2011 Wolfson Microelectronics plc - * - * Author: Dimitris Papastamos <dp@opensource.wolfsonmicro.com> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - */ +// SPDX-License-Identifier: GPL-2.0 +// +// Register cache access API +// +// Copyright 2011 Wolfson Microelectronics plc +// +// Author: Dimitris Papastamos <dp@opensource.wolfsonmicro.com> -#include <linux/slab.h> -#include <linux/export.h> -#include <linux/device.h> -#include <trace/events/regmap.h> #include <linux/bsearch.h> +#include <linux/device.h> +#include <linux/export.h> +#include <linux/slab.h> #include <linux/sort.h> +#include "trace.h" #include "internal.h" static const struct regcache_ops *cache_types[] = { + ®cache_flat_sparse_ops, ®cache_rbtree_ops, - ®cache_lzo_ops, + ®cache_maple_ops, ®cache_flat_ops, }; +static int regcache_defaults_cmp(const void *a, const void *b) +{ + const struct reg_default *x = a; + const struct reg_default *y = b; + + if (x->reg > y->reg) + return 1; + else if (x->reg < y->reg) + return -1; + else + return 0; +} + +void regcache_sort_defaults(struct reg_default *defaults, unsigned int ndefaults) +{ + sort(defaults, ndefaults, sizeof(*defaults), + regcache_defaults_cmp, NULL); +} +EXPORT_SYMBOL_GPL(regcache_sort_defaults); + static int regcache_hw_init(struct regmap *map) { int i, j; int ret; int count; - unsigned int val; + unsigned int reg, val; void *tmp_buf; if (!map->num_reg_defaults_raw) return -EINVAL; + /* calculate the size of reg_defaults */ + for (count = 0, i = 0; i < map->num_reg_defaults_raw; i++) + if (regmap_readable(map, i * map->reg_stride) && + !regmap_volatile(map, i * map->reg_stride)) + count++; + + /* all registers are unreadable or volatile, so just bypass */ + if (!count) { + map->cache_bypass = true; + return 0; + } + + map->num_reg_defaults = count; + map->reg_defaults = kmalloc_array(count, sizeof(struct reg_default), + GFP_KERNEL); + if (!map->reg_defaults) + return -ENOMEM; + if (!map->reg_defaults_raw) { - u32 cache_bypass = map->cache_bypass; + bool cache_bypass = map->cache_bypass; dev_warn(map->dev, "No cache defaults, reading back from HW\n"); - /* Bypass the cache access till data read from HW*/ - map->cache_bypass = 1; + /* Bypass the cache access till data read from HW */ + map->cache_bypass = true; tmp_buf = kmalloc(map->cache_size_raw, GFP_KERNEL); - if (!tmp_buf) - return -EINVAL; + if (!tmp_buf) { + ret = -ENOMEM; + goto err_free; + } ret = regmap_raw_read(map, 0, tmp_buf, - map->num_reg_defaults_raw); + map->cache_size_raw); map->cache_bypass = cache_bypass; - if (ret < 0) { + if (ret == 0) { + map->reg_defaults_raw = tmp_buf; + map->cache_free = true; + } else { kfree(tmp_buf); - return ret; } - map->reg_defaults_raw = tmp_buf; - map->cache_free = 1; - } - - /* calculate the size of reg_defaults */ - for (count = 0, i = 0; i < map->num_reg_defaults_raw; i++) { - val = regcache_get_val(map, map->reg_defaults_raw, i); - if (regmap_volatile(map, i * map->reg_stride)) - continue; - count++; - } - - map->reg_defaults = kmalloc(count * sizeof(struct reg_default), - GFP_KERNEL); - if (!map->reg_defaults) { - ret = -ENOMEM; - goto err_free; } /* fill the reg_defaults */ - map->num_reg_defaults = count; for (i = 0, j = 0; i < map->num_reg_defaults_raw; i++) { - val = regcache_get_val(map, map->reg_defaults_raw, i); - if (regmap_volatile(map, i * map->reg_stride)) + reg = i * map->reg_stride; + + if (!regmap_readable(map, reg)) continue; - map->reg_defaults[j].reg = i * map->reg_stride; + + if (regmap_volatile(map, reg)) + continue; + + if (map->reg_defaults_raw) { + val = regcache_get_val(map, map->reg_defaults_raw, i); + } else { + bool cache_bypass = map->cache_bypass; + + map->cache_bypass = true; + ret = regmap_read(map, reg, &val); + map->cache_bypass = cache_bypass; + if (ret != 0) { + dev_err(map->dev, "Failed to read %d: %d\n", + reg, ret); + goto err_free; + } + } + + map->reg_defaults[j].reg = reg; map->reg_defaults[j].def = val; j++; } @@ -85,8 +126,7 @@ static int regcache_hw_init(struct regmap *map) return 0; err_free: - if (map->cache_free) - kfree(map->reg_defaults_raw); + kfree(map->reg_defaults); return ret; } @@ -97,21 +137,37 @@ int regcache_init(struct regmap *map, const struct regmap_config *config) int i; void *tmp_buf; - for (i = 0; i < config->num_reg_defaults; i++) - if (config->reg_defaults[i].reg % map->reg_stride) - return -EINVAL; - if (map->cache_type == REGCACHE_NONE) { + if (config->reg_defaults || config->num_reg_defaults_raw) + dev_warn(map->dev, + "No cache used with register defaults set!\n"); + map->cache_bypass = true; return 0; } + if (config->reg_defaults && !config->num_reg_defaults) { + dev_err(map->dev, + "Register defaults are set without the number!\n"); + return -EINVAL; + } + + if (config->num_reg_defaults && !config->reg_defaults) { + dev_err(map->dev, + "Register defaults number are set without the reg!\n"); + return -EINVAL; + } + + for (i = 0; i < config->num_reg_defaults; i++) + if (config->reg_defaults[i].reg % map->reg_stride) + return -EINVAL; + for (i = 0; i < ARRAY_SIZE(cache_types); i++) if (cache_types[i]->type == map->cache_type) break; if (i == ARRAY_SIZE(cache_types)) { - dev_err(map->dev, "Could not match compress type: %d\n", + dev_err(map->dev, "Could not match cache type: %d\n", map->cache_type); return -EINVAL; } @@ -119,10 +175,8 @@ int regcache_init(struct regmap *map, const struct regmap_config *config) map->num_reg_defaults = config->num_reg_defaults; map->num_reg_defaults_raw = config->num_reg_defaults_raw; map->reg_defaults_raw = config->reg_defaults_raw; - map->cache_word_size = DIV_ROUND_UP(config->val_bits, 8); + map->cache_word_size = BITS_TO_BYTES(config->val_bits); map->cache_size_raw = map->cache_word_size * config->num_reg_defaults_raw; - map->cache_present = NULL; - map->cache_present_nbits = 0; map->cache = NULL; map->cache_ops = cache_types[i]; @@ -137,10 +191,8 @@ int regcache_init(struct regmap *map, const struct regmap_config *config) * a copy of it. */ if (config->reg_defaults) { - if (!map->num_reg_defaults) - return -EINVAL; - tmp_buf = kmemdup(config->reg_defaults, map->num_reg_defaults * - sizeof(struct reg_default), GFP_KERNEL); + tmp_buf = kmemdup_array(config->reg_defaults, map->num_reg_defaults, + sizeof(*map->reg_defaults), GFP_KERNEL); if (!tmp_buf) return -ENOMEM; map->reg_defaults = tmp_buf; @@ -152,20 +204,42 @@ int regcache_init(struct regmap *map, const struct regmap_config *config) ret = regcache_hw_init(map); if (ret < 0) return ret; + if (map->cache_bypass) + return 0; } - if (!map->max_register) - map->max_register = map->num_reg_defaults_raw; + if (!map->max_register_is_set && map->num_reg_defaults_raw) { + map->max_register = (map->num_reg_defaults_raw - 1) * map->reg_stride; + map->max_register_is_set = true; + } if (map->cache_ops->init) { dev_dbg(map->dev, "Initializing %s cache\n", map->cache_ops->name); + map->lock(map->lock_arg); ret = map->cache_ops->init(map); + map->unlock(map->lock_arg); if (ret) goto err_free; } + + if (map->num_reg_defaults && map->cache_ops->populate) { + dev_dbg(map->dev, "Populating %s cache\n", map->cache_ops->name); + map->lock(map->lock_arg); + ret = map->cache_ops->populate(map); + map->unlock(map->lock_arg); + if (ret) + goto err_exit; + } return 0; +err_exit: + if (map->cache_ops->exit) { + dev_dbg(map->dev, "Destroying %s cache\n", map->cache_ops->name); + map->lock(map->lock_arg); + ret = map->cache_ops->exit(map); + map->unlock(map->lock_arg); + } err_free: kfree(map->reg_defaults); if (map->cache_free) @@ -181,7 +255,6 @@ void regcache_exit(struct regmap *map) BUG_ON(!map->cache_ops); - kfree(map->cache_present); kfree(map->reg_defaults); if (map->cache_free) kfree(map->reg_defaults_raw); @@ -189,12 +262,14 @@ void regcache_exit(struct regmap *map) if (map->cache_ops->exit) { dev_dbg(map->dev, "Destroying %s cache\n", map->cache_ops->name); + map->lock(map->lock_arg); map->cache_ops->exit(map); + map->unlock(map->lock_arg); } } /** - * regcache_read: Fetch the value of a given register from the cache. + * regcache_read - Fetch the value of a given register from the cache. * * @map: map to configure. * @reg: The register index. @@ -208,7 +283,7 @@ int regcache_read(struct regmap *map, int ret; if (map->cache_type == REGCACHE_NONE) - return -ENOSYS; + return -EINVAL; BUG_ON(!map->cache_ops); @@ -216,7 +291,7 @@ int regcache_read(struct regmap *map, ret = map->cache_ops->read(map, reg, value); if (ret == 0) - trace_regmap_reg_read_cache(map->dev, reg, *value); + trace_regmap_reg_read_cache(map, reg, *value); return ret; } @@ -225,7 +300,7 @@ int regcache_read(struct regmap *map, } /** - * regcache_write: Set the value of a given register in the cache. + * regcache_write - Set the value of a given register in the cache. * * @map: map to configure. * @reg: The register index. @@ -241,49 +316,74 @@ int regcache_write(struct regmap *map, BUG_ON(!map->cache_ops); - if (!regmap_writeable(map, reg)) - return -EIO; - if (!regmap_volatile(map, reg)) return map->cache_ops->write(map, reg, value); return 0; } +bool regcache_reg_needs_sync(struct regmap *map, unsigned int reg, + unsigned int val) +{ + int ret; + + if (!regmap_writeable(map, reg)) + return false; + + /* If we don't know the chip just got reset, then sync everything. */ + if (!map->no_sync_defaults) + return true; + + /* Is this the hardware default? If so skip. */ + ret = regcache_lookup_reg(map, reg); + if (ret >= 0 && val == map->reg_defaults[ret].def) + return false; + return true; +} + static int regcache_default_sync(struct regmap *map, unsigned int min, unsigned int max) { unsigned int reg; - for (reg = min; reg <= max; reg++) { + for (reg = min; reg <= max; reg += map->reg_stride) { unsigned int val; int ret; - if (regmap_volatile(map, reg)) + if (regmap_volatile(map, reg) || + !regmap_writeable(map, reg)) continue; ret = regcache_read(map, reg, &val); + if (ret == -ENOENT) + continue; if (ret) return ret; - /* Is this the hardware default? If so skip. */ - ret = regcache_lookup_reg(map, reg); - if (ret >= 0 && val == map->reg_defaults[ret].def) + if (!regcache_reg_needs_sync(map, reg, val)) continue; - map->cache_bypass = 1; + map->cache_bypass = true; ret = _regmap_write(map, reg, val); - map->cache_bypass = 0; - if (ret) + map->cache_bypass = false; + if (ret) { + dev_err(map->dev, "Unable to sync register %#x. %d\n", + reg, ret); return ret; + } dev_dbg(map->dev, "Synced register %#x, value %#x\n", reg, val); } return 0; } +static int rbtree_all(const void *key, const struct rb_node *node) +{ + return 0; +} + /** - * regcache_sync: Sync the register cache with the hardware. + * regcache_sync - Sync the register cache with the hardware. * * @map: map to configure. * @@ -298,7 +398,11 @@ int regcache_sync(struct regmap *map) int ret = 0; unsigned int i; const char *name; - unsigned int bypass; + bool bypass; + struct rb_node *node; + + if (WARN_ON(map->cache_type == REGCACHE_NONE)) + return -EINVAL; BUG_ON(!map->cache_ops); @@ -308,18 +412,14 @@ int regcache_sync(struct regmap *map) dev_dbg(map->dev, "Syncing %s cache\n", map->cache_ops->name); name = map->cache_ops->name; - trace_regcache_sync(map->dev, name, "start"); + trace_regcache_sync(map, name, "start"); if (!map->cache_dirty) goto out; /* Apply any patch first */ - map->cache_bypass = 1; + map->cache_bypass = true; for (i = 0; i < map->patch_regs; i++) { - if (map->patch[i].reg % map->reg_stride) { - ret = -EINVAL; - goto out; - } ret = _regmap_write(map, map->patch[i].reg, map->patch[i].def); if (ret != 0) { dev_err(map->dev, "Failed to write %x = %x: %d\n", @@ -327,7 +427,7 @@ int regcache_sync(struct regmap *map) goto out; } } - map->cache_bypass = 0; + map->cache_bypass = false; if (map->cache_ops->sync) ret = map->cache_ops->sync(map, 0, map->max_register); @@ -338,17 +438,44 @@ int regcache_sync(struct regmap *map) map->cache_dirty = false; out: - trace_regcache_sync(map->dev, name, "stop"); /* Restore the bypass state */ map->cache_bypass = bypass; + map->no_sync_defaults = false; + + /* + * If we did any paging with cache bypassed and a cached + * paging register then the register and cache state might + * have gone out of sync, force writes of all the paging + * registers. + */ + rb_for_each(node, NULL, &map->range_tree, rbtree_all) { + struct regmap_range_node *this = + rb_entry(node, struct regmap_range_node, node); + + /* If there's nothing in the cache there's nothing to sync */ + if (regcache_read(map, this->selector_reg, &i) != 0) + continue; + + ret = _regmap_write(map, this->selector_reg, i); + if (ret != 0) { + dev_err(map->dev, "Failed to write %x = %x: %d\n", + this->selector_reg, i, ret); + break; + } + } + map->unlock(map->lock_arg); + regmap_async_complete(map); + + trace_regcache_sync(map, name, "stop"); + return ret; } EXPORT_SYMBOL_GPL(regcache_sync); /** - * regcache_sync_region: Sync part of the register cache with the hardware. + * regcache_sync_region - Sync part of the register cache with the hardware. * * @map: map to sync. * @min: first register to sync @@ -364,7 +491,10 @@ int regcache_sync_region(struct regmap *map, unsigned int min, { int ret = 0; const char *name; - unsigned int bypass; + bool bypass; + + if (WARN_ON(map->cache_type == REGCACHE_NONE)) + return -EINVAL; BUG_ON(!map->cache_ops); @@ -376,28 +506,35 @@ int regcache_sync_region(struct regmap *map, unsigned int min, name = map->cache_ops->name; dev_dbg(map->dev, "Syncing %s cache from %d-%d\n", name, min, max); - trace_regcache_sync(map->dev, name, "start region"); + trace_regcache_sync(map, name, "start region"); if (!map->cache_dirty) goto out; + map->async = true; + if (map->cache_ops->sync) ret = map->cache_ops->sync(map, min, max); else ret = regcache_default_sync(map, min, max); out: - trace_regcache_sync(map->dev, name, "stop region"); /* Restore the bypass state */ map->cache_bypass = bypass; + map->async = false; + map->no_sync_defaults = false; map->unlock(map->lock_arg); + regmap_async_complete(map); + + trace_regcache_sync(map, name, "stop region"); + return ret; } EXPORT_SYMBOL_GPL(regcache_sync_region); /** - * regcache_drop_region: Discard part of the register cache + * regcache_drop_region - Discard part of the register cache * * @map: map to operate on * @min: first register to discard @@ -410,22 +547,16 @@ EXPORT_SYMBOL_GPL(regcache_sync_region); int regcache_drop_region(struct regmap *map, unsigned int min, unsigned int max) { - unsigned int reg; int ret = 0; - if (!map->cache_present && !(map->cache_ops && map->cache_ops->drop)) + if (!map->cache_ops || !map->cache_ops->drop) return -EINVAL; map->lock(map->lock_arg); - trace_regcache_drop_region(map->dev, min, max); - - if (map->cache_present) - for (reg = min; reg < max + 1; reg++) - clear_bit(reg, map->cache_present); + trace_regcache_drop_region(map, min, max); - if (map->cache_ops && map->cache_ops->drop) - ret = map->cache_ops->drop(map, min, max); + ret = map->cache_ops->drop(map, min, max); map->unlock(map->lock_arg); @@ -434,10 +565,10 @@ int regcache_drop_region(struct regmap *map, unsigned int min, EXPORT_SYMBOL_GPL(regcache_drop_region); /** - * regcache_cache_only: Put a register map into cache only mode + * regcache_cache_only - Put a register map into cache only mode * * @map: map to configure - * @cache_only: flag if changes should be written to the hardware + * @enable: flag if changes should be written to the hardware * * When a register map is marked as cache only writes to the register * map API will only update the register cache, they will not cause @@ -448,38 +579,44 @@ EXPORT_SYMBOL_GPL(regcache_drop_region); void regcache_cache_only(struct regmap *map, bool enable) { map->lock(map->lock_arg); - WARN_ON(map->cache_bypass && enable); + WARN_ON(map->cache_type != REGCACHE_NONE && + map->cache_bypass && enable); map->cache_only = enable; - trace_regmap_cache_only(map->dev, enable); + trace_regmap_cache_only(map, enable); map->unlock(map->lock_arg); } EXPORT_SYMBOL_GPL(regcache_cache_only); /** - * regcache_mark_dirty: Mark the register cache as dirty + * regcache_mark_dirty - Indicate that HW registers were reset to default values * * @map: map to mark * - * Mark the register cache as dirty, for example due to the device - * having been powered down for suspend. If the cache is not marked - * as dirty then the cache sync will be suppressed. + * Inform regcache that the device has been powered down or reset, so that + * on resume, regcache_sync() knows to write out all non-default values + * stored in the cache. + * + * If this function is not called, regcache_sync() will assume that + * the hardware state still matches the cache state, modulo any writes that + * happened when cache_only was true. */ void regcache_mark_dirty(struct regmap *map) { map->lock(map->lock_arg); map->cache_dirty = true; + map->no_sync_defaults = true; map->unlock(map->lock_arg); } EXPORT_SYMBOL_GPL(regcache_mark_dirty); /** - * regcache_cache_bypass: Put a register map into cache bypass mode + * regcache_cache_bypass - Put a register map into cache bypass mode * * @map: map to configure - * @cache_bypass: flag if changes should not be written to the hardware + * @enable: flag if changes should not be written to the cache * * When a register map is marked with the cache bypass option, writes - * to the register map API will only update the hardware and not the + * to the register map API will only update the hardware and not * the cache directly. This is useful when syncing the cache back to * the hardware. */ @@ -488,80 +625,66 @@ void regcache_cache_bypass(struct regmap *map, bool enable) map->lock(map->lock_arg); WARN_ON(map->cache_only && enable); map->cache_bypass = enable; - trace_regmap_cache_bypass(map->dev, enable); + trace_regmap_cache_bypass(map, enable); map->unlock(map->lock_arg); } EXPORT_SYMBOL_GPL(regcache_cache_bypass); -int regcache_set_reg_present(struct regmap *map, unsigned int reg) +/** + * regcache_reg_cached - Check if a register is cached + * + * @map: map to check + * @reg: register to check + * + * Reports if a register is cached. + */ +bool regcache_reg_cached(struct regmap *map, unsigned int reg) { - unsigned long *cache_present; - unsigned int cache_present_size; - unsigned int nregs; - int i; + unsigned int val; + int ret; - nregs = reg + 1; - cache_present_size = BITS_TO_LONGS(nregs); - cache_present_size *= sizeof(long); + map->lock(map->lock_arg); - if (!map->cache_present) { - cache_present = kmalloc(cache_present_size, GFP_KERNEL); - if (!cache_present) - return -ENOMEM; - bitmap_zero(cache_present, nregs); - map->cache_present = cache_present; - map->cache_present_nbits = nregs; - } + ret = regcache_read(map, reg, &val); - if (nregs > map->cache_present_nbits) { - cache_present = krealloc(map->cache_present, - cache_present_size, GFP_KERNEL); - if (!cache_present) - return -ENOMEM; - for (i = 0; i < nregs; i++) - if (i >= map->cache_present_nbits) - clear_bit(i, cache_present); - map->cache_present = cache_present; - map->cache_present_nbits = nregs; - } + map->unlock(map->lock_arg); - set_bit(reg, map->cache_present); - return 0; + return ret == 0; } +EXPORT_SYMBOL_GPL(regcache_reg_cached); -bool regcache_set_val(struct regmap *map, void *base, unsigned int idx, +void regcache_set_val(struct regmap *map, void *base, unsigned int idx, unsigned int val) { - if (regcache_get_val(map, base, idx) == val) - return true; - /* Use device native format if possible */ if (map->format.format_val) { map->format.format_val(base + (map->cache_word_size * idx), val, 0); - return false; + return; } switch (map->cache_word_size) { case 1: { u8 *cache = base; + cache[idx] = val; break; } case 2: { u16 *cache = base; + cache[idx] = val; break; } case 4: { u32 *cache = base; + cache[idx] = val; break; } default: BUG(); } - return false; } unsigned int regcache_get_val(struct regmap *map, const void *base, @@ -578,14 +701,17 @@ unsigned int regcache_get_val(struct regmap *map, const void *base, switch (map->cache_word_size) { case 1: { const u8 *cache = base; + return cache[idx]; } case 2: { const u16 *cache = base; + return cache[idx]; } case 4: { const u32 *cache = base; + return cache[idx]; } default: @@ -620,7 +746,40 @@ int regcache_lookup_reg(struct regmap *map, unsigned int reg) return -ENOENT; } +static bool regcache_reg_present(unsigned long *cache_present, unsigned int idx) +{ + if (!cache_present) + return true; + + return test_bit(idx, cache_present); +} + +int regcache_sync_val(struct regmap *map, unsigned int reg, unsigned int val) +{ + int ret; + + if (!regcache_reg_needs_sync(map, reg, val)) + return 0; + + map->cache_bypass = true; + + ret = _regmap_write(map, reg, val); + + map->cache_bypass = false; + + if (ret != 0) { + dev_err(map->dev, "Unable to sync register %#x. %d\n", + reg, ret); + return ret; + } + dev_dbg(map->dev, "Synced register %#x, value %#x\n", + reg, val); + + return 0; +} + static int regcache_sync_block_single(struct regmap *map, void *block, + unsigned long *cache_present, unsigned int block_base, unsigned int start, unsigned int end) { @@ -630,25 +789,14 @@ static int regcache_sync_block_single(struct regmap *map, void *block, for (i = start; i < end; i++) { regtmp = block_base + (i * map->reg_stride); - if (!regcache_reg_present(map, regtmp)) + if (!regcache_reg_present(cache_present, i) || + !regmap_writeable(map, regtmp)) continue; val = regcache_get_val(map, block, i); - - /* Is this the hardware default? If so skip. */ - ret = regcache_lookup_reg(map, regtmp); - if (ret >= 0 && val == map->reg_defaults[ret].def) - continue; - - map->cache_bypass = 1; - - ret = _regmap_write(map, regtmp, val); - - map->cache_bypass = 0; + ret = regcache_sync_val(map, regtmp, val); if (ret != 0) return ret; - dev_dbg(map->dev, "Synced register %#x, value %#x\n", - regtmp, val); } return 0; @@ -663,17 +811,19 @@ static int regcache_sync_block_raw_flush(struct regmap *map, const void **data, if (*data == NULL) return 0; - count = cur - base; + count = (cur - base) / map->reg_stride; dev_dbg(map->dev, "Writing %zu bytes for %d registers from 0x%x-0x%x\n", - count * val_bytes, count, base, cur - 1); + count * val_bytes, count, base, cur - map->reg_stride); - map->cache_bypass = 1; + map->cache_bypass = true; - ret = _regmap_raw_write(map, base, *data, count * val_bytes, - false); + ret = _regmap_raw_write(map, base, *data, count * val_bytes, false); + if (ret) + dev_err(map->dev, "Unable to sync registers %#x-%#x. %d\n", + base, cur - map->reg_stride, ret); - map->cache_bypass = 0; + map->cache_bypass = false; *data = NULL; @@ -681,6 +831,7 @@ static int regcache_sync_block_raw_flush(struct regmap *map, const void **data, } static int regcache_sync_block_raw(struct regmap *map, void *block, + unsigned long *cache_present, unsigned int block_base, unsigned int start, unsigned int end) { @@ -693,7 +844,8 @@ static int regcache_sync_block_raw(struct regmap *map, void *block, for (i = start; i < end; i++) { regtmp = block_base + (i * map->reg_stride); - if (!regcache_reg_present(map, regtmp)) { + if (!regcache_reg_present(cache_present, i) || + !regmap_writeable(map, regtmp)) { ret = regcache_sync_block_raw_flush(map, &data, base, regtmp); if (ret != 0) @@ -702,10 +854,7 @@ static int regcache_sync_block_raw(struct regmap *map, void *block, } val = regcache_get_val(map, block, i); - - /* Is this the hardware default? If so skip. */ - ret = regcache_lookup_reg(map, regtmp); - if (ret >= 0 && val == map->reg_defaults[ret].def) { + if (!regcache_reg_needs_sync(map, regtmp, val)) { ret = regcache_sync_block_raw_flush(map, &data, base, regtmp); if (ret != 0) @@ -719,17 +868,19 @@ static int regcache_sync_block_raw(struct regmap *map, void *block, } } - return regcache_sync_block_raw_flush(map, &data, base, regtmp); + return regcache_sync_block_raw_flush(map, &data, base, regtmp + + map->reg_stride); } int regcache_sync_block(struct regmap *map, void *block, + unsigned long *cache_present, unsigned int block_base, unsigned int start, unsigned int end) { - if (regmap_can_raw_write(map)) - return regcache_sync_block_raw(map, block, block_base, - start, end); + if (regmap_can_raw_write(map) && !map->use_single_write) + return regcache_sync_block_raw(map, block, cache_present, + block_base, start, end); else - return regcache_sync_block_single(map, block, block_base, - start, end); + return regcache_sync_block_single(map, block, cache_present, + block_base, start, end); } diff --git a/drivers/base/regmap/regmap-ac97.c b/drivers/base/regmap/regmap-ac97.c new file mode 100644 index 000000000000..a561971c459c --- /dev/null +++ b/drivers/base/regmap/regmap-ac97.c @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: GPL-2.0 +// +// Register map access API - AC'97 support +// +// Copyright 2013 Linaro Ltd. All rights reserved. + +#include <linux/clk.h> +#include <linux/err.h> +#include <linux/init.h> +#include <linux/io.h> +#include <linux/module.h> +#include <linux/regmap.h> +#include <linux/slab.h> + +#include <sound/ac97_codec.h> + +bool regmap_ac97_default_volatile(struct device *dev, unsigned int reg) +{ + switch (reg) { + case AC97_RESET: + case AC97_POWERDOWN: + case AC97_INT_PAGING: + case AC97_EXTENDED_ID: + case AC97_EXTENDED_STATUS: + case AC97_EXTENDED_MID: + case AC97_EXTENDED_MSTATUS: + case AC97_GPIO_STATUS: + case AC97_MISC_AFE: + case AC97_VENDOR_ID1: + case AC97_VENDOR_ID2: + case AC97_CODEC_CLASS_REV: + case AC97_PCI_SVID: + case AC97_PCI_SID: + case AC97_FUNC_SELECT: + case AC97_FUNC_INFO: + case AC97_SENSE_INFO: + return true; + default: + return false; + } +} +EXPORT_SYMBOL_GPL(regmap_ac97_default_volatile); + +static int regmap_ac97_reg_read(void *context, unsigned int reg, + unsigned int *val) +{ + struct snd_ac97 *ac97 = context; + + *val = ac97->bus->ops->read(ac97, reg); + + return 0; +} + +static int regmap_ac97_reg_write(void *context, unsigned int reg, + unsigned int val) +{ + struct snd_ac97 *ac97 = context; + + ac97->bus->ops->write(ac97, reg, val); + + return 0; +} + +static const struct regmap_bus ac97_regmap_bus = { + .reg_write = regmap_ac97_reg_write, + .reg_read = regmap_ac97_reg_read, +}; + +struct regmap *__regmap_init_ac97(struct snd_ac97 *ac97, + const struct regmap_config *config, + struct lock_class_key *lock_key, + const char *lock_name) +{ + return __regmap_init(&ac97->dev, &ac97_regmap_bus, ac97, config, + lock_key, lock_name); +} +EXPORT_SYMBOL_GPL(__regmap_init_ac97); + +struct regmap *__devm_regmap_init_ac97(struct snd_ac97 *ac97, + const struct regmap_config *config, + struct lock_class_key *lock_key, + const char *lock_name) +{ + return __devm_regmap_init(&ac97->dev, &ac97_regmap_bus, ac97, config, + lock_key, lock_name); +} +EXPORT_SYMBOL_GPL(__devm_regmap_init_ac97); + +MODULE_DESCRIPTION("Register map access API - AC'97 support"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/base/regmap/regmap-debugfs.c b/drivers/base/regmap/regmap-debugfs.c index 53495753fbdb..c9b4c04b1cf6 100644 --- a/drivers/base/regmap/regmap-debugfs.c +++ b/drivers/base/regmap/regmap-debugfs.c @@ -1,30 +1,34 @@ -/* - * Register map access API - debugfs - * - * Copyright 2011 Wolfson Microelectronics plc - * - * Author: Mark Brown <broonie@opensource.wolfsonmicro.com> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - */ +// SPDX-License-Identifier: GPL-2.0 +// +// Register map access API - debugfs +// +// Copyright 2011 Wolfson Microelectronics plc +// +// Author: Mark Brown <broonie@opensource.wolfsonmicro.com> #include <linux/slab.h> #include <linux/mutex.h> #include <linux/debugfs.h> #include <linux/uaccess.h> #include <linux/device.h> +#include <linux/list.h> #include "internal.h" +struct regmap_debugfs_node { + struct regmap *map; + struct list_head link; +}; + +static unsigned int dummy_index; static struct dentry *regmap_debugfs_root; +static LIST_HEAD(regmap_debugfs_early_list); +static DEFINE_MUTEX(regmap_debugfs_early_lock); /* Calculate the length of a fixed format */ -static size_t regmap_calc_reg_len(int max_val, char *buf, size_t buf_size) +static size_t regmap_calc_reg_len(int max_val) { - snprintf(buf, buf_size, "%x", max_val); - return strlen(buf); + return snprintf(NULL, 0, "%x", max_val); } static ssize_t regmap_name_read_file(struct file *file, @@ -32,6 +36,7 @@ static ssize_t regmap_name_read_file(struct file *file, loff_t *ppos) { struct regmap *map = file->private_data; + const char *name = "nodev"; int ret; char *buf; @@ -39,8 +44,11 @@ static ssize_t regmap_name_read_file(struct file *file, if (!buf) return -ENOMEM; - ret = snprintf(buf, PAGE_SIZE, "%s\n", map->dev->driver->name); - if (ret < 0) { + if (map->dev && map->dev->driver) + name = map->dev->driver->name; + + ret = snprintf(buf, PAGE_SIZE, "%s\n", name); + if (ret >= PAGE_SIZE) { kfree(buf); return ret; } @@ -69,6 +77,17 @@ static void regmap_debugfs_free_dump_cache(struct regmap *map) } } +static bool regmap_printable(struct regmap *map, unsigned int reg) +{ + if (regmap_precious(map, reg)) + return false; + + if (!regmap_readable(map, reg) && !regmap_cached(map, reg)) + return false; + + return true; +} + /* * Work out where the start offset maps into register numbers, bearing * in mind that we suppress hidden registers. @@ -85,8 +104,8 @@ static unsigned int regmap_debugfs_get_dump_start(struct regmap *map, unsigned int reg_offset; /* Suppress the cache if we're using a subrange */ - if (from) - return from; + if (base) + return base; /* * If we don't have a cache build one so we don't have to do a @@ -97,8 +116,7 @@ static unsigned int regmap_debugfs_get_dump_start(struct regmap *map, if (list_empty(&map->debugfs_off_cache)) { for (; i <= map->max_register; i += map->reg_stride) { /* Skip unprinted registers, closing off cache entry */ - if (!regmap_readable(map, i) || - regmap_precious(map, i)) { + if (!regmap_printable(map, i)) { if (c) { c->max = p - 1; c->max_reg = i - map->reg_stride; @@ -165,14 +183,35 @@ static inline void regmap_calc_tot_len(struct regmap *map, { /* Calculate the length of a fixed format */ if (!map->debugfs_tot_len) { - map->debugfs_reg_len = regmap_calc_reg_len(map->max_register, - buf, count); + map->debugfs_reg_len = regmap_calc_reg_len(map->max_register); map->debugfs_val_len = 2 * map->format.val_bytes; map->debugfs_tot_len = map->debugfs_reg_len + map->debugfs_val_len + 3; /* : \n */ } } +static int regmap_next_readable_reg(struct regmap *map, int reg) +{ + struct regmap_debugfs_off_cache *c; + int ret = -EINVAL; + + if (regmap_printable(map, reg + map->reg_stride)) { + ret = reg + map->reg_stride; + } else { + mutex_lock(&map->cache_lock); + list_for_each_entry(c, &map->debugfs_off_cache, list) { + if (reg > c->max_reg) + continue; + if (reg < c->base_reg) { + ret = c->base_reg; + break; + } + } + mutex_unlock(&map->cache_lock); + } + return ret; +} + static ssize_t regmap_read_debugfs(struct regmap *map, unsigned int from, unsigned int to, char __user *user_buf, size_t count, loff_t *ppos) @@ -187,6 +226,9 @@ static ssize_t regmap_read_debugfs(struct regmap *map, unsigned int from, if (*ppos < 0 || !count) return -EINVAL; + if (count > (PAGE_SIZE << MAX_PAGE_ORDER)) + count = PAGE_SIZE << MAX_PAGE_ORDER; + buf = kmalloc(count, GFP_KERNEL); if (!buf) return -ENOMEM; @@ -196,12 +238,8 @@ static ssize_t regmap_read_debugfs(struct regmap *map, unsigned int from, /* Work out which register we're starting at */ start_reg = regmap_debugfs_get_dump_start(map, from, *ppos, &p); - for (i = start_reg; i <= to; i += map->reg_stride) { - if (!regmap_readable(map, i)) - continue; - - if (regmap_precious(map, i)) - continue; + for (i = start_reg; i >= 0 && i <= to; + i = regmap_next_readable_reg(map, i)) { /* If we're in the region the user is trying to read */ if (p >= *ppos) { @@ -281,7 +319,7 @@ static ssize_t regmap_map_write_file(struct file *file, reg = simple_strtoul(start, &start, 16); while (*start == ' ') start++; - if (strict_strtoul(start, 16, &value)) + if (kstrtoul(start, 16, &value)) return -EINVAL; /* Userspace has been fiddling around behind the kernel's back */ @@ -330,10 +368,14 @@ static ssize_t regmap_reg_ranges_read_file(struct file *file, char *buf; char *entry; int ret; + unsigned int entry_len; if (*ppos < 0 || !count) return -EINVAL; + if (count > (PAGE_SIZE << MAX_PAGE_ORDER)) + count = PAGE_SIZE << MAX_PAGE_ORDER; + buf = kmalloc(count, GFP_KERNEL); if (!buf) return -ENOMEM; @@ -357,18 +399,15 @@ static ssize_t regmap_reg_ranges_read_file(struct file *file, p = 0; mutex_lock(&map->cache_lock); list_for_each_entry(c, &map->debugfs_off_cache, list) { - snprintf(entry, PAGE_SIZE, "%x-%x", - c->base_reg, c->max_reg); + entry_len = snprintf(entry, PAGE_SIZE, "%x-%x\n", + c->base_reg, c->max_reg); if (p >= *ppos) { - if (buf_pos + 1 + strlen(entry) > count) + if (buf_pos + entry_len > count) break; - snprintf(buf + buf_pos, count - buf_pos, - "%s", entry); - buf_pos += strlen(entry); - buf[buf_pos] = '\n'; - buf_pos++; + memcpy(buf + buf_pos, entry, entry_len); + buf_pos += entry_len; } - p += strlen(entry) + 1; + p += entry_len; } mutex_unlock(&map->cache_lock); @@ -392,118 +431,212 @@ static const struct file_operations regmap_reg_ranges_fops = { .llseek = default_llseek, }; -static ssize_t regmap_access_read_file(struct file *file, - char __user *user_buf, size_t count, - loff_t *ppos) +static int regmap_access_show(struct seq_file *s, void *ignored) { - int reg_len, tot_len; - size_t buf_pos = 0; - loff_t p = 0; - ssize_t ret; - int i; - struct regmap *map = file->private_data; - char *buf; - - if (*ppos < 0 || !count) - return -EINVAL; - - buf = kmalloc(count, GFP_KERNEL); - if (!buf) - return -ENOMEM; + struct regmap *map = s->private; + int i, reg_len; - /* Calculate the length of a fixed format */ - reg_len = regmap_calc_reg_len(map->max_register, buf, count); - tot_len = reg_len + 10; /* ': R W V P\n' */ + reg_len = regmap_calc_reg_len(map->max_register); for (i = 0; i <= map->max_register; i += map->reg_stride) { /* Ignore registers which are neither readable nor writable */ if (!regmap_readable(map, i) && !regmap_writeable(map, i)) continue; - /* If we're in the region the user is trying to read */ - if (p >= *ppos) { - /* ...but not beyond it */ - if (buf_pos >= count - 1 - tot_len) - break; + /* Format the register */ + seq_printf(s, "%.*x: %c %c %c %c\n", reg_len, i, + regmap_readable(map, i) ? 'y' : 'n', + regmap_writeable(map, i) ? 'y' : 'n', + regmap_volatile(map, i) ? 'y' : 'n', + regmap_precious(map, i) ? 'y' : 'n'); + } - /* Format the register */ - snprintf(buf + buf_pos, count - buf_pos, - "%.*x: %c %c %c %c\n", - reg_len, i, - regmap_readable(map, i) ? 'y' : 'n', - regmap_writeable(map, i) ? 'y' : 'n', - regmap_volatile(map, i) ? 'y' : 'n', - regmap_precious(map, i) ? 'y' : 'n'); - - buf_pos += tot_len; - } - p += tot_len; + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(regmap_access); + +static ssize_t regmap_cache_only_write_file(struct file *file, + const char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct regmap *map = container_of(file->private_data, + struct regmap, cache_only); + bool new_val, require_sync = false; + int err; + + err = kstrtobool_from_user(user_buf, count, &new_val); + /* Ignore malforned data like debugfs_write_file_bool() */ + if (err) + return count; + + map->lock(map->lock_arg); + + if (new_val && !map->cache_only) { + dev_warn(map->dev, "debugfs cache_only=Y forced\n"); + add_taint(TAINT_USER, LOCKDEP_STILL_OK); + } else if (!new_val && map->cache_only) { + dev_warn(map->dev, "debugfs cache_only=N forced: syncing cache\n"); + require_sync = true; } + map->cache_only = new_val; - ret = buf_pos; + map->unlock(map->lock_arg); - if (copy_to_user(user_buf, buf, buf_pos)) { - ret = -EFAULT; - goto out; + if (require_sync) { + err = regcache_sync(map); + if (err) + dev_err(map->dev, "Failed to sync cache %d\n", err); } - *ppos += buf_pos; + return count; +} -out: - kfree(buf); - return ret; +static const struct file_operations regmap_cache_only_fops = { + .open = simple_open, + .read = debugfs_read_file_bool, + .write = regmap_cache_only_write_file, +}; + +static ssize_t regmap_cache_bypass_write_file(struct file *file, + const char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct regmap *map = container_of(file->private_data, + struct regmap, cache_bypass); + bool new_val; + int err; + + err = kstrtobool_from_user(user_buf, count, &new_val); + /* Ignore malforned data like debugfs_write_file_bool() */ + if (err) + return count; + + map->lock(map->lock_arg); + + if (new_val && !map->cache_bypass) { + dev_warn(map->dev, "debugfs cache_bypass=Y forced\n"); + add_taint(TAINT_USER, LOCKDEP_STILL_OK); + } else if (!new_val && map->cache_bypass) { + dev_warn(map->dev, "debugfs cache_bypass=N forced\n"); + } + map->cache_bypass = new_val; + + map->unlock(map->lock_arg); + + return count; } -static const struct file_operations regmap_access_fops = { +static const struct file_operations regmap_cache_bypass_fops = { .open = simple_open, - .read = regmap_access_read_file, - .llseek = default_llseek, + .read = debugfs_read_file_bool, + .write = regmap_cache_bypass_write_file, }; -void regmap_debugfs_init(struct regmap *map, const char *name) +void regmap_debugfs_init(struct regmap *map) { struct rb_node *next; struct regmap_range_node *range_node; + const char *devname = "dummy"; + const char *name = map->name; + + /* + * Userspace can initiate reads from the hardware over debugfs. + * Normally internal regmap structures and buffers are protected with + * a mutex or a spinlock, but if the regmap owner decided to disable + * all locking mechanisms, this is no longer the case. For safety: + * don't create the debugfs entries if locking is disabled. + */ + if (map->debugfs_disable) { + dev_dbg(map->dev, "regmap locking disabled - not creating debugfs entries\n"); + return; + } + + /* If we don't have the debugfs root yet, postpone init */ + if (!regmap_debugfs_root) { + struct regmap_debugfs_node *node; + node = kzalloc(sizeof(*node), GFP_KERNEL); + if (!node) + return; + node->map = map; + mutex_lock(®map_debugfs_early_lock); + list_add(&node->link, ®map_debugfs_early_list); + mutex_unlock(®map_debugfs_early_lock); + return; + } INIT_LIST_HEAD(&map->debugfs_off_cache); mutex_init(&map->cache_lock); + if (map->dev) + devname = dev_name(map->dev); + if (name) { - map->debugfs_name = kasprintf(GFP_KERNEL, "%s-%s", - dev_name(map->dev), name); + if (!map->debugfs_name) { + map->debugfs_name = kasprintf(GFP_KERNEL, "%s-%s", + devname, name); + if (!map->debugfs_name) + return; + } name = map->debugfs_name; } else { - name = dev_name(map->dev); + name = devname; } - map->debugfs = debugfs_create_dir(name, regmap_debugfs_root); - if (!map->debugfs) { - dev_warn(map->dev, "Failed to create debugfs directory\n"); - return; + if (!strcmp(name, "dummy")) { + kfree(map->debugfs_name); + map->debugfs_name = kasprintf(GFP_KERNEL, "dummy%d", + dummy_index); + if (!map->debugfs_name) + return; + name = map->debugfs_name; + dummy_index++; } + map->debugfs = debugfs_create_dir(name, regmap_debugfs_root); + debugfs_create_file("name", 0400, map->debugfs, map, ®map_name_fops); debugfs_create_file("range", 0400, map->debugfs, map, ®map_reg_ranges_fops); - if (map->max_register) { - debugfs_create_file("registers", 0400, map->debugfs, + if (map->max_register || regmap_readable(map, 0)) { + umode_t registers_mode; + +#if defined(REGMAP_ALLOW_WRITE_DEBUGFS) + registers_mode = 0600; +#else + registers_mode = 0400; +#endif + + debugfs_create_file("registers", registers_mode, map->debugfs, map, ®map_map_fops); debugfs_create_file("access", 0400, map->debugfs, map, ®map_access_fops); } if (map->cache_type) { - debugfs_create_bool("cache_only", 0400, map->debugfs, - &map->cache_only); + debugfs_create_file("cache_only", 0600, map->debugfs, + &map->cache_only, ®map_cache_only_fops); debugfs_create_bool("cache_dirty", 0400, map->debugfs, &map->cache_dirty); - debugfs_create_bool("cache_bypass", 0400, map->debugfs, - &map->cache_bypass); + debugfs_create_file("cache_bypass", 0600, map->debugfs, + &map->cache_bypass, + ®map_cache_bypass_fops); } + /* + * This could interfere with driver operation. Therefore, don't provide + * any real compile time configuration option for this feature. One will + * have to modify the source code directly in order to use it. + */ +#undef REGMAP_ALLOW_FORCE_WRITE_FIELD_DEBUGFS +#ifdef REGMAP_ALLOW_FORCE_WRITE_FIELD_DEBUGFS + debugfs_create_bool("force_write_field", 0600, map->debugfs, + &map->force_write_field); +#endif + next = rb_first(&map->range_tree); while (next) { range_node = rb_entry(next, struct regmap_range_node, node); @@ -515,22 +648,46 @@ void regmap_debugfs_init(struct regmap *map, const char *name) next = rb_next(&range_node->node); } + + if (map->cache_ops && map->cache_ops->debugfs_init) + map->cache_ops->debugfs_init(map); } void regmap_debugfs_exit(struct regmap *map) { - debugfs_remove_recursive(map->debugfs); - mutex_lock(&map->cache_lock); - regmap_debugfs_free_dump_cache(map); - mutex_unlock(&map->cache_lock); - kfree(map->debugfs_name); + if (map->debugfs) { + debugfs_remove_recursive(map->debugfs); + mutex_lock(&map->cache_lock); + regmap_debugfs_free_dump_cache(map); + mutex_unlock(&map->cache_lock); + kfree(map->debugfs_name); + map->debugfs_name = NULL; + } else { + struct regmap_debugfs_node *node, *tmp; + + mutex_lock(®map_debugfs_early_lock); + list_for_each_entry_safe(node, tmp, ®map_debugfs_early_list, + link) { + if (node->map == map) { + list_del(&node->link); + kfree(node); + } + } + mutex_unlock(®map_debugfs_early_lock); + } } void regmap_debugfs_initcall(void) { + struct regmap_debugfs_node *node, *tmp; + regmap_debugfs_root = debugfs_create_dir("regmap", NULL); - if (!regmap_debugfs_root) { - pr_warn("regmap: Failed to create debugfs root\n"); - return; + + mutex_lock(®map_debugfs_early_lock); + list_for_each_entry_safe(node, tmp, ®map_debugfs_early_list, link) { + regmap_debugfs_init(node->map); + list_del(&node->link); + kfree(node); } + mutex_unlock(®map_debugfs_early_lock); } diff --git a/drivers/base/regmap/regmap-fsi.c b/drivers/base/regmap/regmap-fsi.c new file mode 100644 index 000000000000..3d2f3cb31d5e --- /dev/null +++ b/drivers/base/regmap/regmap-fsi.c @@ -0,0 +1,231 @@ +// SPDX-License-Identifier: GPL-2.0 +// +// Register map access API - FSI support +// +// Copyright 2022 IBM Corp +// +// Author: Eddie James <eajames@linux.ibm.com> + +#include <linux/fsi.h> +#include <linux/module.h> +#include <linux/regmap.h> + +#include "internal.h" + +static int regmap_fsi32_reg_read(void *context, unsigned int reg, unsigned int *val) +{ + u32 v; + int ret; + + ret = fsi_slave_read(context, reg, &v, sizeof(v)); + if (ret) + return ret; + + *val = v; + return 0; +} + +static int regmap_fsi32_reg_write(void *context, unsigned int reg, unsigned int val) +{ + u32 v = val; + + return fsi_slave_write(context, reg, &v, sizeof(v)); +} + +static const struct regmap_bus regmap_fsi32 = { + .reg_write = regmap_fsi32_reg_write, + .reg_read = regmap_fsi32_reg_read, +}; + +static int regmap_fsi32le_reg_read(void *context, unsigned int reg, unsigned int *val) +{ + __be32 v; + int ret; + + ret = fsi_slave_read(context, reg, &v, sizeof(v)); + if (ret) + return ret; + + *val = be32_to_cpu(v); + return 0; +} + +static int regmap_fsi32le_reg_write(void *context, unsigned int reg, unsigned int val) +{ + __be32 v = cpu_to_be32(val); + + return fsi_slave_write(context, reg, &v, sizeof(v)); +} + +static const struct regmap_bus regmap_fsi32le = { + .reg_write = regmap_fsi32le_reg_write, + .reg_read = regmap_fsi32le_reg_read, +}; + +static int regmap_fsi16_reg_read(void *context, unsigned int reg, unsigned int *val) +{ + u16 v; + int ret; + + ret = fsi_slave_read(context, reg, &v, sizeof(v)); + if (ret) + return ret; + + *val = v; + return 0; +} + +static int regmap_fsi16_reg_write(void *context, unsigned int reg, unsigned int val) +{ + u16 v; + + if (val > 0xffff) + return -EINVAL; + + v = val; + return fsi_slave_write(context, reg, &v, sizeof(v)); +} + +static const struct regmap_bus regmap_fsi16 = { + .reg_write = regmap_fsi16_reg_write, + .reg_read = regmap_fsi16_reg_read, +}; + +static int regmap_fsi16le_reg_read(void *context, unsigned int reg, unsigned int *val) +{ + __be16 v; + int ret; + + ret = fsi_slave_read(context, reg, &v, sizeof(v)); + if (ret) + return ret; + + *val = be16_to_cpu(v); + return 0; +} + +static int regmap_fsi16le_reg_write(void *context, unsigned int reg, unsigned int val) +{ + __be16 v; + + if (val > 0xffff) + return -EINVAL; + + v = cpu_to_be16(val); + return fsi_slave_write(context, reg, &v, sizeof(v)); +} + +static const struct regmap_bus regmap_fsi16le = { + .reg_write = regmap_fsi16le_reg_write, + .reg_read = regmap_fsi16le_reg_read, +}; + +static int regmap_fsi8_reg_read(void *context, unsigned int reg, unsigned int *val) +{ + u8 v; + int ret; + + ret = fsi_slave_read(context, reg, &v, sizeof(v)); + if (ret) + return ret; + + *val = v; + return 0; +} + +static int regmap_fsi8_reg_write(void *context, unsigned int reg, unsigned int val) +{ + u8 v; + + if (val > 0xff) + return -EINVAL; + + v = val; + return fsi_slave_write(context, reg, &v, sizeof(v)); +} + +static const struct regmap_bus regmap_fsi8 = { + .reg_write = regmap_fsi8_reg_write, + .reg_read = regmap_fsi8_reg_read, +}; + +static const struct regmap_bus *regmap_get_fsi_bus(struct fsi_device *fsi_dev, + const struct regmap_config *config) +{ + const struct regmap_bus *bus = NULL; + + if (config->reg_bits == 8 || config->reg_bits == 16 || config->reg_bits == 32) { + switch (config->val_bits) { + case 8: + bus = ®map_fsi8; + break; + case 16: + switch (regmap_get_val_endian(&fsi_dev->dev, NULL, config)) { + case REGMAP_ENDIAN_LITTLE: +#ifdef __LITTLE_ENDIAN + case REGMAP_ENDIAN_NATIVE: +#endif + bus = ®map_fsi16le; + break; + case REGMAP_ENDIAN_DEFAULT: + case REGMAP_ENDIAN_BIG: +#ifdef __BIG_ENDIAN + case REGMAP_ENDIAN_NATIVE: +#endif + bus = ®map_fsi16; + break; + default: + break; + } + break; + case 32: + switch (regmap_get_val_endian(&fsi_dev->dev, NULL, config)) { + case REGMAP_ENDIAN_LITTLE: +#ifdef __LITTLE_ENDIAN + case REGMAP_ENDIAN_NATIVE: +#endif + bus = ®map_fsi32le; + break; + case REGMAP_ENDIAN_DEFAULT: + case REGMAP_ENDIAN_BIG: +#ifdef __BIG_ENDIAN + case REGMAP_ENDIAN_NATIVE: +#endif + bus = ®map_fsi32; + break; + default: + break; + } + break; + } + } + + return bus ?: ERR_PTR(-EOPNOTSUPP); +} + +struct regmap *__regmap_init_fsi(struct fsi_device *fsi_dev, const struct regmap_config *config, + struct lock_class_key *lock_key, const char *lock_name) +{ + const struct regmap_bus *bus = regmap_get_fsi_bus(fsi_dev, config); + + if (IS_ERR(bus)) + return ERR_CAST(bus); + + return __regmap_init(&fsi_dev->dev, bus, fsi_dev->slave, config, lock_key, lock_name); +} +EXPORT_SYMBOL_GPL(__regmap_init_fsi); + +struct regmap *__devm_regmap_init_fsi(struct fsi_device *fsi_dev, + const struct regmap_config *config, + struct lock_class_key *lock_key, const char *lock_name) +{ + const struct regmap_bus *bus = regmap_get_fsi_bus(fsi_dev, config); + + if (IS_ERR(bus)) + return ERR_CAST(bus); + + return __devm_regmap_init(&fsi_dev->dev, bus, fsi_dev->slave, config, lock_key, lock_name); +} +EXPORT_SYMBOL_GPL(__devm_regmap_init_fsi); + +MODULE_LICENSE("GPL"); diff --git a/drivers/base/regmap/regmap-i2c.c b/drivers/base/regmap/regmap-i2c.c index fa6bf5279d28..c9b39a02278e 100644 --- a/drivers/base/regmap/regmap-i2c.c +++ b/drivers/base/regmap/regmap-i2c.c @@ -1,19 +1,124 @@ -/* - * Register map access API - I2C support - * - * Copyright 2011 Wolfson Microelectronics plc - * - * Author: Mark Brown <broonie@opensource.wolfsonmicro.com> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - */ +// SPDX-License-Identifier: GPL-2.0 +// +// Register map access API - I2C support +// +// Copyright 2011 Wolfson Microelectronics plc +// +// Author: Mark Brown <broonie@opensource.wolfsonmicro.com> #include <linux/regmap.h> #include <linux/i2c.h> #include <linux/module.h> -#include <linux/init.h> + +#include "internal.h" + +static int regmap_smbus_byte_reg_read(void *context, unsigned int reg, + unsigned int *val) +{ + struct device *dev = context; + struct i2c_client *i2c = to_i2c_client(dev); + int ret; + + if (reg > 0xff) + return -EINVAL; + + ret = i2c_smbus_read_byte_data(i2c, reg); + if (ret < 0) + return ret; + + *val = ret; + + return 0; +} + +static int regmap_smbus_byte_reg_write(void *context, unsigned int reg, + unsigned int val) +{ + struct device *dev = context; + struct i2c_client *i2c = to_i2c_client(dev); + + if (val > 0xff || reg > 0xff) + return -EINVAL; + + return i2c_smbus_write_byte_data(i2c, reg, val); +} + +static const struct regmap_bus regmap_smbus_byte = { + .reg_write = regmap_smbus_byte_reg_write, + .reg_read = regmap_smbus_byte_reg_read, +}; + +static int regmap_smbus_word_reg_read(void *context, unsigned int reg, + unsigned int *val) +{ + struct device *dev = context; + struct i2c_client *i2c = to_i2c_client(dev); + int ret; + + if (reg > 0xff) + return -EINVAL; + + ret = i2c_smbus_read_word_data(i2c, reg); + if (ret < 0) + return ret; + + *val = ret; + + return 0; +} + +static int regmap_smbus_word_reg_write(void *context, unsigned int reg, + unsigned int val) +{ + struct device *dev = context; + struct i2c_client *i2c = to_i2c_client(dev); + + if (val > 0xffff || reg > 0xff) + return -EINVAL; + + return i2c_smbus_write_word_data(i2c, reg, val); +} + +static const struct regmap_bus regmap_smbus_word = { + .reg_write = regmap_smbus_word_reg_write, + .reg_read = regmap_smbus_word_reg_read, +}; + +static int regmap_smbus_word_read_swapped(void *context, unsigned int reg, + unsigned int *val) +{ + struct device *dev = context; + struct i2c_client *i2c = to_i2c_client(dev); + int ret; + + if (reg > 0xff) + return -EINVAL; + + ret = i2c_smbus_read_word_swapped(i2c, reg); + if (ret < 0) + return ret; + + *val = ret; + + return 0; +} + +static int regmap_smbus_word_write_swapped(void *context, unsigned int reg, + unsigned int val) +{ + struct device *dev = context; + struct i2c_client *i2c = to_i2c_client(dev); + + if (val > 0xffff || reg > 0xff) + return -EINVAL; + + return i2c_smbus_write_word_swapped(i2c, reg, val); +} + +static const struct regmap_bus regmap_smbus_word_swapped = { + .reg_write = regmap_smbus_word_write_swapped, + .reg_read = regmap_smbus_word_read_swapped, +}; static int regmap_i2c_write(void *context, const void *data, size_t count) { @@ -92,43 +197,205 @@ static int regmap_i2c_read(void *context, return -EIO; } -static struct regmap_bus regmap_i2c = { +static const struct regmap_bus regmap_i2c = { .write = regmap_i2c_write, .gather_write = regmap_i2c_gather_write, .read = regmap_i2c_read, + .reg_format_endian_default = REGMAP_ENDIAN_BIG, + .val_format_endian_default = REGMAP_ENDIAN_BIG, +}; + +static int regmap_i2c_smbus_i2c_write(void *context, const void *data, + size_t count) +{ + struct device *dev = context; + struct i2c_client *i2c = to_i2c_client(dev); + + if (count < 1) + return -EINVAL; + + --count; + return i2c_smbus_write_i2c_block_data(i2c, ((u8 *)data)[0], count, + ((u8 *)data + 1)); +} + +static int regmap_i2c_smbus_i2c_read(void *context, const void *reg, + size_t reg_size, void *val, + size_t val_size) +{ + struct device *dev = context; + struct i2c_client *i2c = to_i2c_client(dev); + int ret; + + if (reg_size != 1 || val_size < 1) + return -EINVAL; + + ret = i2c_smbus_read_i2c_block_data(i2c, ((u8 *)reg)[0], val_size, val); + if (ret == val_size) + return 0; + else if (ret < 0) + return ret; + else + return -EIO; +} + +static const struct regmap_bus regmap_i2c_smbus_i2c_block = { + .write = regmap_i2c_smbus_i2c_write, + .read = regmap_i2c_smbus_i2c_read, + .max_raw_read = I2C_SMBUS_BLOCK_MAX - 1, + .max_raw_write = I2C_SMBUS_BLOCK_MAX - 1, }; -/** - * regmap_init_i2c(): Initialise register map - * - * @i2c: Device that will be interacted with - * @config: Configuration for register map - * - * The return value will be an ERR_PTR() on error or a valid pointer to - * a struct regmap. - */ -struct regmap *regmap_init_i2c(struct i2c_client *i2c, - const struct regmap_config *config) -{ - return regmap_init(&i2c->dev, ®map_i2c, &i2c->dev, config); -} -EXPORT_SYMBOL_GPL(regmap_init_i2c); - -/** - * devm_regmap_init_i2c(): Initialise managed register map - * - * @i2c: Device that will be interacted with - * @config: Configuration for register map - * - * The return value will be an ERR_PTR() on error or a valid pointer - * to a struct regmap. The regmap will be automatically freed by the - * device management code. - */ -struct regmap *devm_regmap_init_i2c(struct i2c_client *i2c, - const struct regmap_config *config) -{ - return devm_regmap_init(&i2c->dev, ®map_i2c, &i2c->dev, config); -} -EXPORT_SYMBOL_GPL(devm_regmap_init_i2c); +static int regmap_i2c_smbus_i2c_write_reg16(void *context, const void *data, + size_t count) +{ + struct device *dev = context; + struct i2c_client *i2c = to_i2c_client(dev); + + if (count < 2) + return -EINVAL; + + count--; + return i2c_smbus_write_i2c_block_data(i2c, ((u8 *)data)[0], count, + (u8 *)data + 1); +} + +static int regmap_i2c_smbus_i2c_read_reg16(void *context, const void *reg, + size_t reg_size, void *val, + size_t val_size) +{ + struct device *dev = context; + struct i2c_client *i2c = to_i2c_client(dev); + int ret, count, len = val_size; + + if (reg_size != 2) + return -EINVAL; + + ret = i2c_smbus_write_byte_data(i2c, ((u16 *)reg)[0] & 0xff, + ((u16 *)reg)[0] >> 8); + if (ret < 0) + return ret; + + count = 0; + do { + /* Current Address Read */ + ret = i2c_smbus_read_byte(i2c); + if (ret < 0) + break; + + *((u8 *)val++) = ret; + count++; + len--; + } while (len > 0); + + if (count == val_size) + return 0; + else if (ret < 0) + return ret; + else + return -EIO; +} + +static const struct regmap_bus regmap_i2c_smbus_i2c_block_reg16 = { + .write = regmap_i2c_smbus_i2c_write_reg16, + .read = regmap_i2c_smbus_i2c_read_reg16, + .max_raw_read = I2C_SMBUS_BLOCK_MAX - 2, + .max_raw_write = I2C_SMBUS_BLOCK_MAX - 2, +}; + +static const struct regmap_bus *regmap_get_i2c_bus(struct i2c_client *i2c, + const struct regmap_config *config) +{ + const struct i2c_adapter_quirks *quirks; + const struct regmap_bus *bus = NULL; + struct regmap_bus *ret_bus; + u16 max_read = 0, max_write = 0; + + if (i2c_check_functionality(i2c->adapter, I2C_FUNC_I2C)) + bus = ®map_i2c; + else if (config->val_bits == 8 && config->reg_bits == 8 && + i2c_check_functionality(i2c->adapter, + I2C_FUNC_SMBUS_I2C_BLOCK)) + bus = ®map_i2c_smbus_i2c_block; + else if (config->val_bits == 8 && config->reg_bits == 16 && + i2c_check_functionality(i2c->adapter, + I2C_FUNC_SMBUS_I2C_BLOCK)) + bus = ®map_i2c_smbus_i2c_block_reg16; + else if (config->val_bits == 16 && config->reg_bits == 8 && + i2c_check_functionality(i2c->adapter, + I2C_FUNC_SMBUS_WORD_DATA)) + switch (regmap_get_val_endian(&i2c->dev, NULL, config)) { + case REGMAP_ENDIAN_LITTLE: + bus = ®map_smbus_word; + break; + case REGMAP_ENDIAN_BIG: + bus = ®map_smbus_word_swapped; + break; + default: /* everything else is not supported */ + break; + } + else if (config->val_bits == 8 && config->reg_bits == 8 && + i2c_check_functionality(i2c->adapter, + I2C_FUNC_SMBUS_BYTE_DATA)) + bus = ®map_smbus_byte; + + if (!bus) + return ERR_PTR(-ENOTSUPP); + + quirks = i2c->adapter->quirks; + if (quirks) { + if (quirks->max_read_len && + (bus->max_raw_read == 0 || bus->max_raw_read > quirks->max_read_len)) + max_read = quirks->max_read_len; + + if (quirks->max_write_len && + (bus->max_raw_write == 0 || bus->max_raw_write > quirks->max_write_len)) + max_write = quirks->max_write_len - + (config->reg_bits + config->pad_bits) / BITS_PER_BYTE; + + if (max_read || max_write) { + ret_bus = kmemdup(bus, sizeof(*bus), GFP_KERNEL); + if (!ret_bus) + return ERR_PTR(-ENOMEM); + ret_bus->free_on_exit = true; + ret_bus->max_raw_read = max_read; + ret_bus->max_raw_write = max_write; + bus = ret_bus; + } + } + + return bus; +} + +struct regmap *__regmap_init_i2c(struct i2c_client *i2c, + const struct regmap_config *config, + struct lock_class_key *lock_key, + const char *lock_name) +{ + const struct regmap_bus *bus = regmap_get_i2c_bus(i2c, config); + + if (IS_ERR(bus)) + return ERR_CAST(bus); + + return __regmap_init(&i2c->dev, bus, &i2c->dev, config, + lock_key, lock_name); +} +EXPORT_SYMBOL_GPL(__regmap_init_i2c); + +struct regmap *__devm_regmap_init_i2c(struct i2c_client *i2c, + const struct regmap_config *config, + struct lock_class_key *lock_key, + const char *lock_name) +{ + const struct regmap_bus *bus = regmap_get_i2c_bus(i2c, config); + + if (IS_ERR(bus)) + return ERR_CAST(bus); + + return __devm_regmap_init(&i2c->dev, bus, &i2c->dev, config, + lock_key, lock_name); +} +EXPORT_SYMBOL_GPL(__devm_regmap_init_i2c); +MODULE_DESCRIPTION("Register map access API - I2C support"); MODULE_LICENSE("GPL"); diff --git a/drivers/base/regmap/regmap-i3c.c b/drivers/base/regmap/regmap-i3c.c new file mode 100644 index 000000000000..863b348704dc --- /dev/null +++ b/drivers/base/regmap/regmap-i3c.c @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) 2018 Synopsys, Inc. and/or its affiliates. + +#include <linux/array_size.h> +#include <linux/regmap.h> +#include <linux/i3c/device.h> +#include <linux/i3c/master.h> +#include <linux/module.h> + +static int regmap_i3c_write(void *context, const void *data, size_t count) +{ + struct device *dev = context; + struct i3c_device *i3c = dev_to_i3cdev(dev); + struct i3c_xfer xfers[] = { + { + .rnw = false, + .len = count, + .data.out = data, + }, + }; + + return i3c_device_do_xfers(i3c, xfers, ARRAY_SIZE(xfers), I3C_SDR); +} + +static int regmap_i3c_read(void *context, + const void *reg, size_t reg_size, + void *val, size_t val_size) +{ + struct device *dev = context; + struct i3c_device *i3c = dev_to_i3cdev(dev); + struct i3c_xfer xfers[2]; + + xfers[0].rnw = false; + xfers[0].len = reg_size; + xfers[0].data.out = reg; + + xfers[1].rnw = true; + xfers[1].len = val_size; + xfers[1].data.in = val; + + return i3c_device_do_xfers(i3c, xfers, ARRAY_SIZE(xfers), I3C_SDR); +} + +static const struct regmap_bus regmap_i3c = { + .write = regmap_i3c_write, + .read = regmap_i3c_read, +}; + +struct regmap *__devm_regmap_init_i3c(struct i3c_device *i3c, + const struct regmap_config *config, + struct lock_class_key *lock_key, + const char *lock_name) +{ + return __devm_regmap_init(&i3c->dev, ®map_i3c, &i3c->dev, config, + lock_key, lock_name); +} +EXPORT_SYMBOL_GPL(__devm_regmap_init_i3c); + +MODULE_AUTHOR("Vitor Soares <vitor.soares@synopsys.com>"); +MODULE_DESCRIPTION("regmap I3C Module"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/base/regmap/regmap-irq.c b/drivers/base/regmap/regmap-irq.c index 1643e889bafc..6112d942499b 100644 --- a/drivers/base/regmap/regmap-irq.c +++ b/drivers/base/regmap/regmap-irq.c @@ -1,28 +1,27 @@ -/* - * regmap based irq_chip - * - * Copyright 2011 Wolfson Microelectronics plc - * - * Author: Mark Brown <broonie@opensource.wolfsonmicro.com> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - */ - -#include <linux/export.h> +// SPDX-License-Identifier: GPL-2.0 +// +// regmap based irq_chip +// +// Copyright 2011 Wolfson Microelectronics plc +// +// Author: Mark Brown <broonie@opensource.wolfsonmicro.com> + +#include <linux/array_size.h> #include <linux/device.h> -#include <linux/regmap.h> -#include <linux/irq.h> +#include <linux/export.h> #include <linux/interrupt.h> +#include <linux/irq.h> #include <linux/irqdomain.h> +#include <linux/overflow.h> #include <linux/pm_runtime.h> +#include <linux/regmap.h> #include <linux/slab.h> #include "internal.h" struct regmap_irq_chip_data { struct mutex lock; + struct lock_class_key lock_key; struct irq_chip irq_chip; struct regmap *map; @@ -35,12 +34,22 @@ struct regmap_irq_chip_data { int wake_count; void *status_reg_buf; + unsigned int *main_status_buf; unsigned int *status_buf; + unsigned int *prev_status_buf; unsigned int *mask_buf; unsigned int *mask_buf_def; unsigned int *wake_buf; + unsigned int *type_buf; + unsigned int *type_buf_def; + unsigned int **config_buf; unsigned int irq_reg_stride; + + unsigned int (*get_irq_reg)(struct regmap_irq_chip_data *data, + unsigned int base, int index); + + unsigned int clear_status:1; }; static inline const @@ -50,6 +59,20 @@ struct regmap_irq *irq_to_regmap_irq(struct regmap_irq_chip_data *data, return &data->chip->irqs[irq]; } +static bool regmap_irq_can_bulk_read_status(struct regmap_irq_chip_data *data) +{ + struct regmap *map = data->map; + + /* + * While possible that a user-defined ->get_irq_reg() callback might + * be linear enough to support bulk reads, most of the time it won't. + * Therefore only allow them if the default callback is being used. + */ + return data->irq_reg_stride == 1 && map->reg_stride == 1 && + data->get_irq_reg == regmap_irq_get_irq_reg_linear && + !map->use_single_read; +} + static void regmap_irq_lock(struct irq_data *data) { struct regmap_irq_chip_data *d = irq_data_get_irq_chip_data(data); @@ -61,8 +84,9 @@ static void regmap_irq_sync_unlock(struct irq_data *data) { struct regmap_irq_chip_data *d = irq_data_get_irq_chip_data(data); struct regmap *map = d->map; - int i, ret; + int i, j, ret; u32 reg; + u32 val; if (d->chip->runtime_pm) { ret = pm_runtime_get_sync(map->dev); @@ -71,26 +95,49 @@ static void regmap_irq_sync_unlock(struct irq_data *data) ret); } + if (d->clear_status) { + for (i = 0; i < d->chip->num_regs; i++) { + reg = d->get_irq_reg(d, d->chip->status_base, i); + + ret = regmap_read(map, reg, &val); + if (ret) + dev_err(d->map->dev, + "Failed to clear the interrupt status bits\n"); + } + + d->clear_status = false; + } + /* * If there's been a change in the mask write it back to the * hardware. We rely on the use of the regmap core cache to * suppress pointless writes. */ for (i = 0; i < d->chip->num_regs; i++) { - reg = d->chip->mask_base + - (i * map->reg_stride * d->irq_reg_stride); - if (d->chip->mask_invert) + if (d->chip->handle_mask_sync) + d->chip->handle_mask_sync(i, d->mask_buf_def[i], + d->mask_buf[i], + d->chip->irq_drv_data); + + if (d->chip->mask_base && !d->chip->handle_mask_sync) { + reg = d->get_irq_reg(d, d->chip->mask_base, i); ret = regmap_update_bits(d->map, reg, - d->mask_buf_def[i], ~d->mask_buf[i]); - else + d->mask_buf_def[i], + d->mask_buf[i]); + if (ret) + dev_err(d->map->dev, "Failed to sync masks in %x\n", reg); + } + + if (d->chip->unmask_base && !d->chip->handle_mask_sync) { + reg = d->get_irq_reg(d, d->chip->unmask_base, i); ret = regmap_update_bits(d->map, reg, - d->mask_buf_def[i], d->mask_buf[i]); - if (ret != 0) - dev_err(d->map->dev, "Failed to sync masks in %x\n", - reg); + d->mask_buf_def[i], ~d->mask_buf[i]); + if (ret) + dev_err(d->map->dev, "Failed to sync masks in %x\n", + reg); + } - reg = d->chip->wake_base + - (i * map->reg_stride * d->irq_reg_stride); + reg = d->get_irq_reg(d, d->chip->wake_base, i); if (d->wake_buf) { if (d->chip->wake_invert) ret = regmap_update_bits(d->map, reg, @@ -105,6 +152,43 @@ static void regmap_irq_sync_unlock(struct irq_data *data) "Failed to sync wakes in %x: %d\n", reg, ret); } + + if (!d->chip->init_ack_masked) + continue; + /* + * Ack all the masked interrupts unconditionally, + * OR if there is masked interrupt which hasn't been Acked, + * it'll be ignored in irq handler, then may introduce irq storm + */ + if (d->mask_buf[i] && (d->chip->ack_base || d->chip->use_ack)) { + reg = d->get_irq_reg(d, d->chip->ack_base, i); + + /* some chips ack by write 0 */ + if (d->chip->ack_invert) + ret = regmap_write(map, reg, ~d->mask_buf[i]); + else + ret = regmap_write(map, reg, d->mask_buf[i]); + if (d->chip->clear_ack) { + if (d->chip->ack_invert && !ret) + ret = regmap_write(map, reg, UINT_MAX); + else if (!ret) + ret = regmap_write(map, reg, 0); + } + if (ret != 0) + dev_err(d->map->dev, "Failed to ack 0x%x: %d\n", + reg, ret); + } + } + + for (i = 0; i < d->chip->num_config_bases; i++) { + for (j = 0; j < d->chip->num_config_regs; j++) { + reg = d->get_irq_reg(d, d->chip->config_base[i], j); + ret = regmap_write(map, reg, d->config_buf[i][j]); + if (ret) + dev_err(d->map->dev, + "Failed to write config %x: %d\n", + reg, ret); + } } if (d->chip->runtime_pm) @@ -113,10 +197,10 @@ static void regmap_irq_sync_unlock(struct irq_data *data) /* If we've changed our wakeup count propagate it to the parent */ if (d->wake_count < 0) for (i = d->wake_count; i < 0; i++) - irq_set_irq_wake(d->irq, 0); + disable_irq_wake(d->irq); else if (d->wake_count > 0) for (i = 0; i < d->wake_count; i++) - irq_set_irq_wake(d->irq, 1); + enable_irq_wake(d->irq); d->wake_count = 0; @@ -128,8 +212,28 @@ static void regmap_irq_enable(struct irq_data *data) struct regmap_irq_chip_data *d = irq_data_get_irq_chip_data(data); struct regmap *map = d->map; const struct regmap_irq *irq_data = irq_to_regmap_irq(d, data->hwirq); + unsigned int reg = irq_data->reg_offset / map->reg_stride; + unsigned int mask; + + /* + * The type_in_mask flag means that the underlying hardware uses + * separate mask bits for each interrupt trigger type, but we want + * to have a single logical interrupt with a configurable type. + * + * If the interrupt we're enabling defines any supported types + * then instead of using the regular mask bits for this interrupt, + * use the value previously written to the type buffer at the + * corresponding offset in regmap_irq_set_type(). + */ + if (d->chip->type_in_mask && irq_data->type.types_supported) + mask = d->type_buf[reg] & irq_data->mask; + else + mask = irq_data->mask; + + if (d->chip->clear_on_unmask) + d->clear_status = true; - d->mask_buf[irq_data->reg_offset / map->reg_stride] &= ~irq_data->mask; + d->mask_buf[reg] &= ~mask; } static void regmap_irq_disable(struct irq_data *data) @@ -141,6 +245,36 @@ static void regmap_irq_disable(struct irq_data *data) d->mask_buf[irq_data->reg_offset / map->reg_stride] |= irq_data->mask; } +static int regmap_irq_set_type(struct irq_data *data, unsigned int type) +{ + struct regmap_irq_chip_data *d = irq_data_get_irq_chip_data(data); + struct regmap *map = d->map; + const struct regmap_irq *irq_data = irq_to_regmap_irq(d, data->hwirq); + int reg, ret; + const struct regmap_irq_type *t = &irq_data->type; + + if ((t->types_supported & type) != type) + return 0; + + reg = t->type_reg_offset / map->reg_stride; + + if (d->chip->type_in_mask) { + ret = regmap_irq_set_type_config_simple(&d->type_buf, type, + irq_data, reg, d->chip->irq_drv_data); + if (ret) + return ret; + } + + if (d->chip->set_type_config) { + ret = d->chip->set_type_config(d->config_buf, type, irq_data, + reg, d->chip->irq_drv_data); + if (ret) + return ret; + } + + return 0; +} + static int regmap_irq_set_wake(struct irq_data *data, unsigned int on) { struct regmap_irq_chip_data *d = irq_data_get_irq_chip_data(data); @@ -167,34 +301,99 @@ static const struct irq_chip regmap_irq_chip = { .irq_bus_sync_unlock = regmap_irq_sync_unlock, .irq_disable = regmap_irq_disable, .irq_enable = regmap_irq_enable, + .irq_set_type = regmap_irq_set_type, .irq_set_wake = regmap_irq_set_wake, }; -static irqreturn_t regmap_irq_thread(int irq, void *d) +static inline int read_sub_irq_data(struct regmap_irq_chip_data *data, + unsigned int b) { - struct regmap_irq_chip_data *data = d; const struct regmap_irq_chip *chip = data->chip; + const struct regmap_irq_sub_irq_map *subreg; struct regmap *map = data->map; - int ret, i; - bool handled = false; - u32 reg; + unsigned int reg; + int i, ret = 0; - if (chip->runtime_pm) { - ret = pm_runtime_get_sync(map->dev); - if (ret < 0) { - dev_err(map->dev, "IRQ thread failed to resume: %d\n", - ret); - pm_runtime_put(map->dev); - return IRQ_NONE; + if (!chip->sub_reg_offsets) { + reg = data->get_irq_reg(data, chip->status_base, b); + ret = regmap_read(map, reg, &data->status_buf[b]); + } else { + /* + * Note we can't use ->get_irq_reg() here because the offsets + * in 'subreg' are *not* interchangeable with indices. + */ + subreg = &chip->sub_reg_offsets[b]; + for (i = 0; i < subreg->num_regs; i++) { + unsigned int offset = subreg->offset[i]; + unsigned int index = offset / map->reg_stride; + + ret = regmap_read(map, chip->status_base + offset, + &data->status_buf[index]); + if (ret) + break; } } + return ret; +} + +static int read_irq_data(struct regmap_irq_chip_data *data) +{ + const struct regmap_irq_chip *chip = data->chip; + struct regmap *map = data->map; + int ret, i; + u32 reg; /* - * Read in the statuses, using a single bulk read if possible - * in order to reduce the I/O overheads. + * Read only registers with active IRQs if the chip has 'main status + * register'. Else read in the statuses, using a single bulk read if + * possible in order to reduce the I/O overheads. */ - if (!map->use_single_rw && map->reg_stride == 1 && - data->irq_reg_stride == 1) { + + if (chip->no_status) { + /* no status register so default to all active */ + memset32(data->status_buf, GENMASK(31, 0), chip->num_regs); + } else if (chip->num_main_regs) { + unsigned int max_main_bits; + + max_main_bits = (chip->num_main_status_bits) ? + chip->num_main_status_bits : chip->num_regs; + /* Clear the status buf as we don't read all status regs */ + memset32(data->status_buf, 0, chip->num_regs); + + /* We could support bulk read for main status registers + * but I don't expect to see devices with really many main + * status registers so let's only support single reads for the + * sake of simplicity. and add bulk reads only if needed + */ + for (i = 0; i < chip->num_main_regs; i++) { + reg = data->get_irq_reg(data, chip->main_status, i); + ret = regmap_read(map, reg, &data->main_status_buf[i]); + if (ret) { + dev_err(map->dev, "Failed to read IRQ status %d\n", ret); + return ret; + } + } + + /* Read sub registers with active IRQs */ + for (i = 0; i < chip->num_main_regs; i++) { + unsigned int b; + const unsigned long mreg = data->main_status_buf[i]; + + for_each_set_bit(b, &mreg, map->format.val_bytes * 8) { + if (i * map->format.val_bytes * 8 + b > + max_main_bits) + break; + ret = read_sub_irq_data(data, b); + + if (ret != 0) { + dev_err(map->dev, "Failed to read IRQ status %d\n", ret); + return ret; + } + } + + } + } else if (regmap_irq_can_bulk_read_status(data)) { + u8 *buf8 = data->status_reg_buf; u16 *buf16 = data->status_reg_buf; u32 *buf32 = data->status_reg_buf; @@ -205,9 +404,8 @@ static irqreturn_t regmap_irq_thread(int irq, void *d) data->status_reg_buf, chip->num_regs); if (ret != 0) { - dev_err(map->dev, "Failed to read IRQ status: %d\n", - ret); - return IRQ_NONE; + dev_err(map->dev, "Failed to read IRQ status: %d\n", ret); + return ret; } for (i = 0; i < data->chip->num_regs; i++) { @@ -223,31 +421,66 @@ static irqreturn_t regmap_irq_thread(int irq, void *d) break; default: BUG(); - return IRQ_NONE; + return -EIO; } } } else { for (i = 0; i < data->chip->num_regs; i++) { - ret = regmap_read(map, chip->status_base + - (i * map->reg_stride - * data->irq_reg_stride), - &data->status_buf[i]); + unsigned int reg = data->get_irq_reg(data, + data->chip->status_base, i); + ret = regmap_read(map, reg, &data->status_buf[i]); if (ret != 0) { - dev_err(map->dev, - "Failed to read IRQ status: %d\n", - ret); - if (chip->runtime_pm) - pm_runtime_put(map->dev); - return IRQ_NONE; + dev_err(map->dev, "Failed to read IRQ status: %d\n", ret); + return ret; } } } + if (chip->status_invert) + for (i = 0; i < data->chip->num_regs; i++) + data->status_buf[i] = ~data->status_buf[i]; + + return 0; +} + +static irqreturn_t regmap_irq_thread(int irq, void *d) +{ + struct regmap_irq_chip_data *data = d; + const struct regmap_irq_chip *chip = data->chip; + struct regmap *map = data->map; + int ret, i; + bool handled = false; + u32 reg; + + if (chip->handle_pre_irq) + chip->handle_pre_irq(chip->irq_drv_data); + + if (chip->runtime_pm) { + ret = pm_runtime_get_sync(map->dev); + if (ret < 0) { + dev_err(map->dev, "IRQ thread failed to resume: %d\n", ret); + goto exit; + } + } + + ret = read_irq_data(data); + if (ret < 0) + goto exit; + + if (chip->status_is_level) { + for (i = 0; i < data->chip->num_regs; i++) { + unsigned int val = data->status_buf[i]; + + data->status_buf[i] ^= data->prev_status_buf[i]; + data->prev_status_buf[i] = val; + } + } + /* * Ignore masked IRQs and ack if we need to; we ack early so - * there is no race between handling and acknowleding the + * there is no race between handling and acknowledging the * interrupt. We assume that typically few of the interrupts * will fire simultaneously so don't worry about overhead from * doing a write per register. @@ -255,10 +488,21 @@ static irqreturn_t regmap_irq_thread(int irq, void *d) for (i = 0; i < data->chip->num_regs; i++) { data->status_buf[i] &= ~data->mask_buf[i]; - if (data->status_buf[i] && chip->ack_base) { - reg = chip->ack_base + - (i * map->reg_stride * data->irq_reg_stride); - ret = regmap_write(map, reg, data->status_buf[i]); + if (data->status_buf[i] && (chip->ack_base || chip->use_ack)) { + reg = data->get_irq_reg(data, data->chip->ack_base, i); + + if (chip->ack_invert) + ret = regmap_write(map, reg, + ~data->status_buf[i]); + else + ret = regmap_write(map, reg, + data->status_buf[i]); + if (chip->clear_ack) { + if (chip->ack_invert && !ret) + ret = regmap_write(map, reg, UINT_MAX); + else if (!ret) + ret = regmap_write(map, reg, 0); + } if (ret != 0) dev_err(map->dev, "Failed to ack 0x%x: %d\n", reg, ret); @@ -273,6 +517,10 @@ static irqreturn_t regmap_irq_thread(int irq, void *d) } } +exit: + if (chip->handle_post_irq) + chip->handle_post_irq(chip->irq_drv_data); + if (chip->runtime_pm) pm_runtime_put(map->dev); @@ -282,39 +530,138 @@ static irqreturn_t regmap_irq_thread(int irq, void *d) return IRQ_NONE; } +static struct lock_class_key regmap_irq_lock_class; +static struct lock_class_key regmap_irq_request_class; + static int regmap_irq_map(struct irq_domain *h, unsigned int virq, irq_hw_number_t hw) { struct regmap_irq_chip_data *data = h->host_data; irq_set_chip_data(virq, data); + irq_set_lockdep_class(virq, ®map_irq_lock_class, ®map_irq_request_class); irq_set_chip(virq, &data->irq_chip); irq_set_nested_thread(virq, 1); - - /* ARM needs us to explicitly flag the IRQ as valid - * and will set them noprobe when we do so. */ -#ifdef CONFIG_ARM - set_irq_flags(virq, IRQF_VALID); -#else + irq_set_parent(virq, data->irq); irq_set_noprobe(virq); -#endif return 0; } -static struct irq_domain_ops regmap_domain_ops = { +static const struct irq_domain_ops regmap_domain_ops = { .map = regmap_irq_map, - .xlate = irq_domain_xlate_twocell, + .xlate = irq_domain_xlate_onetwocell, }; /** - * regmap_add_irq_chip(): Use standard regmap IRQ controller handling + * regmap_irq_get_irq_reg_linear() - Linear IRQ register mapping callback. + * @data: Data for the &struct regmap_irq_chip + * @base: Base register + * @index: Register index + * + * Returns the register address corresponding to the given @base and @index + * by the formula ``base + index * regmap_stride * irq_reg_stride``. + */ +unsigned int regmap_irq_get_irq_reg_linear(struct regmap_irq_chip_data *data, + unsigned int base, int index) +{ + struct regmap *map = data->map; + + return base + index * map->reg_stride * data->irq_reg_stride; +} +EXPORT_SYMBOL_GPL(regmap_irq_get_irq_reg_linear); + +/** + * regmap_irq_set_type_config_simple() - Simple IRQ type configuration callback. + * @buf: Buffer containing configuration register values, this is a 2D array of + * `num_config_bases` rows, each of `num_config_regs` elements. + * @type: The requested IRQ type. + * @irq_data: The IRQ being configured. + * @idx: Index of the irq's config registers within each array `buf[i]` + * @irq_drv_data: Driver specific IRQ data + * + * This is a &struct regmap_irq_chip->set_type_config callback suitable for + * chips with one config register. Register values are updated according to + * the &struct regmap_irq_type data associated with an IRQ. + */ +int regmap_irq_set_type_config_simple(unsigned int **buf, unsigned int type, + const struct regmap_irq *irq_data, + int idx, void *irq_drv_data) +{ + const struct regmap_irq_type *t = &irq_data->type; + + if (t->type_reg_mask) + buf[0][idx] &= ~t->type_reg_mask; + else + buf[0][idx] &= ~(t->type_falling_val | + t->type_rising_val | + t->type_level_low_val | + t->type_level_high_val); + + switch (type) { + case IRQ_TYPE_EDGE_FALLING: + buf[0][idx] |= t->type_falling_val; + break; + + case IRQ_TYPE_EDGE_RISING: + buf[0][idx] |= t->type_rising_val; + break; + + case IRQ_TYPE_EDGE_BOTH: + buf[0][idx] |= (t->type_falling_val | + t->type_rising_val); + break; + + case IRQ_TYPE_LEVEL_HIGH: + buf[0][idx] |= t->type_level_high_val; + break; + + case IRQ_TYPE_LEVEL_LOW: + buf[0][idx] |= t->type_level_low_val; + break; + + default: + return -EINVAL; + } + + return 0; +} +EXPORT_SYMBOL_GPL(regmap_irq_set_type_config_simple); + +static int regmap_irq_create_domain(struct fwnode_handle *fwnode, int irq_base, + const struct regmap_irq_chip *chip, + struct regmap_irq_chip_data *d) +{ + struct irq_domain_info info = { + .fwnode = fwnode, + .size = chip->num_irqs, + .hwirq_max = chip->num_irqs, + .virq_base = irq_base, + .ops = ®map_domain_ops, + .host_data = d, + .name_suffix = chip->domain_suffix, + }; + + d->domain = irq_domain_instantiate(&info); + if (IS_ERR(d->domain)) { + dev_err(d->map->dev, "Failed to create IRQ domain\n"); + return PTR_ERR(d->domain); + } + + return 0; +} + + +/** + * regmap_add_irq_chip_fwnode() - Use standard regmap IRQ controller handling * - * map: The regmap for the device. - * irq: The IRQ the device uses to signal interrupts - * irq_flags: The IRQF_ flags to use for the primary interrupt. - * chip: Configuration for the interrupt controller. - * data: Runtime data structure for the controller, allocated on success + * @fwnode: The firmware node where the IRQ domain should be added to. + * @map: The regmap for the device. + * @irq: The IRQ the device uses to signal interrupts. + * @irq_flags: The IRQF_ flags to use for the primary interrupt. + * @irq_base: Allocate at specific IRQ number if irq_base > 0. + * @chip: Configuration for the interrupt controller. + * @data: Runtime data structure for the controller, allocated on success. * * Returns 0 on success or an errno on failure. * @@ -322,15 +669,26 @@ static struct irq_domain_ops regmap_domain_ops = { * register cache. The chip driver is responsible for restoring the * register values used by the IRQ controller over suspend and resume. */ -int regmap_add_irq_chip(struct regmap *map, int irq, int irq_flags, - int irq_base, const struct regmap_irq_chip *chip, - struct regmap_irq_chip_data **data) +int regmap_add_irq_chip_fwnode(struct fwnode_handle *fwnode, + struct regmap *map, int irq, + int irq_flags, int irq_base, + const struct regmap_irq_chip *chip, + struct regmap_irq_chip_data **data) { struct regmap_irq_chip_data *d; int i; int ret = -ENOMEM; u32 reg; + if (chip->num_regs <= 0) + return -EINVAL; + + if (chip->clear_on_unmask && (chip->ack_base || chip->use_ack)) + return -EINVAL; + + if (chip->mask_base && chip->unmask_base && !chip->mask_unmask_non_inverted) + return -EINVAL; + for (i = 0; i < chip->num_irqs; i++) { if (chip->irqs[i].reg_offset % map->reg_stride) return -EINVAL; @@ -352,30 +710,73 @@ int regmap_add_irq_chip(struct regmap *map, int irq, int irq_flags, if (!d) return -ENOMEM; - *data = d; + if (chip->num_main_regs) { + d->main_status_buf = kcalloc(chip->num_main_regs, + sizeof(*d->main_status_buf), + GFP_KERNEL); + + if (!d->main_status_buf) + goto err_alloc; + } - d->status_buf = kzalloc(sizeof(unsigned int) * chip->num_regs, + d->status_buf = kcalloc(chip->num_regs, sizeof(*d->status_buf), GFP_KERNEL); if (!d->status_buf) goto err_alloc; - d->mask_buf = kzalloc(sizeof(unsigned int) * chip->num_regs, + if (chip->status_is_level) { + d->prev_status_buf = kcalloc(chip->num_regs, sizeof(*d->prev_status_buf), + GFP_KERNEL); + if (!d->prev_status_buf) + goto err_alloc; + } + + d->mask_buf = kcalloc(chip->num_regs, sizeof(*d->mask_buf), GFP_KERNEL); if (!d->mask_buf) goto err_alloc; - d->mask_buf_def = kzalloc(sizeof(unsigned int) * chip->num_regs, + d->mask_buf_def = kcalloc(chip->num_regs, sizeof(*d->mask_buf_def), GFP_KERNEL); if (!d->mask_buf_def) goto err_alloc; if (chip->wake_base) { - d->wake_buf = kzalloc(sizeof(unsigned int) * chip->num_regs, + d->wake_buf = kcalloc(chip->num_regs, sizeof(*d->wake_buf), GFP_KERNEL); if (!d->wake_buf) goto err_alloc; } + if (chip->type_in_mask) { + d->type_buf_def = kcalloc(chip->num_regs, + sizeof(*d->type_buf_def), GFP_KERNEL); + if (!d->type_buf_def) + goto err_alloc; + + d->type_buf = kcalloc(chip->num_regs, sizeof(*d->type_buf), GFP_KERNEL); + if (!d->type_buf) + goto err_alloc; + } + + if (chip->num_config_bases && chip->num_config_regs) { + /* + * Create config_buf[num_config_bases][num_config_regs] + */ + d->config_buf = kcalloc(chip->num_config_bases, + sizeof(*d->config_buf), GFP_KERNEL); + if (!d->config_buf) + goto err_alloc; + + for (i = 0; i < chip->num_config_bases; i++) { + d->config_buf[i] = kcalloc(chip->num_config_regs, + sizeof(**d->config_buf), + GFP_KERNEL); + if (!d->config_buf[i]) + goto err_alloc; + } + } + d->irq_chip = regmap_irq_chip; d->irq_chip.name = chip->name; d->irq = irq; @@ -388,15 +789,26 @@ int regmap_add_irq_chip(struct regmap *map, int irq, int irq_flags, else d->irq_reg_stride = 1; - if (!map->use_single_rw && map->reg_stride == 1 && - d->irq_reg_stride == 1) { - d->status_reg_buf = kmalloc(map->format.val_bytes * - chip->num_regs, GFP_KERNEL); + if (chip->get_irq_reg) + d->get_irq_reg = chip->get_irq_reg; + else + d->get_irq_reg = regmap_irq_get_irq_reg_linear; + + if (regmap_irq_can_bulk_read_status(d)) { + d->status_reg_buf = kmalloc_array(chip->num_regs, + map->format.val_bytes, + GFP_KERNEL); if (!d->status_reg_buf) goto err_alloc; } - mutex_init(&d->lock); + /* + * If one regmap-irq is the parent of another then we'll try + * to lock the child with the parent locked, use an explicit + * lock_key so lockdep can figure out what's going on. + */ + lockdep_register_key(&d->lock_key); + mutex_init_with_key(&d->lock, &d->lock_key); for (i = 0; i < chip->num_irqs; i++) d->mask_buf_def[chip->irqs[i].reg_offset / map->reg_stride] @@ -405,18 +817,77 @@ int regmap_add_irq_chip(struct regmap *map, int irq, int irq_flags, /* Mask all the interrupts by default */ for (i = 0; i < chip->num_regs; i++) { d->mask_buf[i] = d->mask_buf_def[i]; - reg = chip->mask_base + - (i * map->reg_stride * d->irq_reg_stride); - if (chip->mask_invert) - ret = regmap_update_bits(map, reg, - d->mask_buf[i], ~d->mask_buf[i]); - else - ret = regmap_update_bits(map, reg, - d->mask_buf[i], d->mask_buf[i]); - if (ret != 0) { - dev_err(map->dev, "Failed to set masks in 0x%x: %d\n", - reg, ret); - goto err_alloc; + + if (chip->handle_mask_sync) { + ret = chip->handle_mask_sync(i, d->mask_buf_def[i], + d->mask_buf[i], + chip->irq_drv_data); + if (ret) + goto err_mutex; + } + + if (chip->mask_base && !chip->handle_mask_sync) { + reg = d->get_irq_reg(d, chip->mask_base, i); + ret = regmap_update_bits(d->map, reg, + d->mask_buf_def[i], + d->mask_buf[i]); + if (ret) { + dev_err(map->dev, "Failed to set masks in 0x%x: %d\n", + reg, ret); + goto err_mutex; + } + } + + if (chip->unmask_base && !chip->handle_mask_sync) { + reg = d->get_irq_reg(d, chip->unmask_base, i); + ret = regmap_update_bits(d->map, reg, + d->mask_buf_def[i], ~d->mask_buf[i]); + if (ret) { + dev_err(map->dev, "Failed to set masks in 0x%x: %d\n", + reg, ret); + goto err_mutex; + } + } + + if (!chip->init_ack_masked) + continue; + + /* Ack masked but set interrupts */ + if (d->chip->no_status) { + /* no status register so default to all active */ + d->status_buf[i] = UINT_MAX; + } else { + reg = d->get_irq_reg(d, d->chip->status_base, i); + ret = regmap_read(map, reg, &d->status_buf[i]); + if (ret != 0) { + dev_err(map->dev, "Failed to read IRQ status: %d\n", + ret); + goto err_mutex; + } + } + + if (chip->status_invert) + d->status_buf[i] = ~d->status_buf[i]; + + if (d->status_buf[i] && (chip->ack_base || chip->use_ack)) { + reg = d->get_irq_reg(d, d->chip->ack_base, i); + if (chip->ack_invert) + ret = regmap_write(map, reg, + ~(d->status_buf[i] & d->mask_buf[i])); + else + ret = regmap_write(map, reg, + d->status_buf[i] & d->mask_buf[i]); + if (chip->clear_ack) { + if (chip->ack_invert && !ret) + ret = regmap_write(map, reg, UINT_MAX); + else if (!ret) + ret = regmap_write(map, reg, 0); + } + if (ret != 0) { + dev_err(map->dev, "Failed to ack 0x%x: %d\n", + reg, ret); + goto err_mutex; + } } } @@ -424,40 +895,40 @@ int regmap_add_irq_chip(struct regmap *map, int irq, int irq_flags, if (d->wake_buf) { for (i = 0; i < chip->num_regs; i++) { d->wake_buf[i] = d->mask_buf_def[i]; - reg = chip->wake_base + - (i * map->reg_stride * d->irq_reg_stride); + reg = d->get_irq_reg(d, d->chip->wake_base, i); if (chip->wake_invert) - ret = regmap_update_bits(map, reg, + ret = regmap_update_bits(d->map, reg, d->mask_buf_def[i], 0); else - ret = regmap_update_bits(map, reg, + ret = regmap_update_bits(d->map, reg, d->mask_buf_def[i], d->wake_buf[i]); if (ret != 0) { dev_err(map->dev, "Failed to set masks in 0x%x: %d\n", reg, ret); - goto err_alloc; + goto err_mutex; } } } - if (irq_base) - d->domain = irq_domain_add_legacy(map->dev->of_node, - chip->num_irqs, irq_base, 0, - ®map_domain_ops, d); - else - d->domain = irq_domain_add_linear(map->dev->of_node, - chip->num_irqs, - ®map_domain_ops, d); - if (!d->domain) { - dev_err(map->dev, "Failed to create IRQ domain\n"); - ret = -ENOMEM; - goto err_alloc; + /* Store current levels */ + if (chip->status_is_level) { + ret = read_irq_data(d); + if (ret < 0) + goto err_mutex; + + memcpy(d->prev_status_buf, d->status_buf, + array_size(d->chip->num_regs, sizeof(d->prev_status_buf[0]))); } - ret = request_threaded_irq(irq, NULL, regmap_irq_thread, irq_flags, + ret = regmap_irq_create_domain(fwnode, irq_base, chip, d); + if (ret) + goto err_mutex; + + ret = request_threaded_irq(irq, NULL, regmap_irq_thread, + irq_flags | IRQF_ONESHOT, chip->name, d); if (ret != 0) { dev_err(map->dev, "Failed to request IRQ %d for %s: %d\n", @@ -465,49 +936,234 @@ int regmap_add_irq_chip(struct regmap *map, int irq, int irq_flags, goto err_domain; } + *data = d; + return 0; err_domain: /* Should really dispose of the domain but... */ +err_mutex: + mutex_destroy(&d->lock); + lockdep_unregister_key(&d->lock_key); err_alloc: + kfree(d->type_buf); + kfree(d->type_buf_def); kfree(d->wake_buf); kfree(d->mask_buf_def); kfree(d->mask_buf); + kfree(d->main_status_buf); kfree(d->status_buf); + kfree(d->prev_status_buf); kfree(d->status_reg_buf); + if (d->config_buf) { + for (i = 0; i < chip->num_config_bases; i++) + kfree(d->config_buf[i]); + kfree(d->config_buf); + } kfree(d); return ret; } +EXPORT_SYMBOL_GPL(regmap_add_irq_chip_fwnode); + +/** + * regmap_add_irq_chip() - Use standard regmap IRQ controller handling + * + * @map: The regmap for the device. + * @irq: The IRQ the device uses to signal interrupts. + * @irq_flags: The IRQF_ flags to use for the primary interrupt. + * @irq_base: Allocate at specific IRQ number if irq_base > 0. + * @chip: Configuration for the interrupt controller. + * @data: Runtime data structure for the controller, allocated on success. + * + * Returns 0 on success or an errno on failure. + * + * This is the same as regmap_add_irq_chip_fwnode, except that the firmware + * node of the regmap is used. + */ +int regmap_add_irq_chip(struct regmap *map, int irq, int irq_flags, + int irq_base, const struct regmap_irq_chip *chip, + struct regmap_irq_chip_data **data) +{ + return regmap_add_irq_chip_fwnode(dev_fwnode(map->dev), map, irq, + irq_flags, irq_base, chip, data); +} EXPORT_SYMBOL_GPL(regmap_add_irq_chip); /** - * regmap_del_irq_chip(): Stop interrupt handling for a regmap IRQ chip + * regmap_del_irq_chip() - Stop interrupt handling for a regmap IRQ chip * * @irq: Primary IRQ for the device - * @d: regmap_irq_chip_data allocated by regmap_add_irq_chip() + * @d: ®map_irq_chip_data allocated by regmap_add_irq_chip() + * + * This function also disposes of all mapped IRQs on the chip. */ void regmap_del_irq_chip(int irq, struct regmap_irq_chip_data *d) { + unsigned int virq; + int i, hwirq; + if (!d) return; free_irq(irq, d); - /* We should unmap the domain but... */ + + /* Dispose all virtual irq from irq domain before removing it */ + for (hwirq = 0; hwirq < d->chip->num_irqs; hwirq++) { + /* Ignore hwirq if holes in the IRQ list */ + if (!d->chip->irqs[hwirq].mask) + continue; + + /* + * Find the virtual irq of hwirq on chip and if it is + * there then dispose it + */ + virq = irq_find_mapping(d->domain, hwirq); + if (virq) + irq_dispose_mapping(virq); + } + + irq_domain_remove(d->domain); + kfree(d->type_buf); + kfree(d->type_buf_def); kfree(d->wake_buf); kfree(d->mask_buf_def); kfree(d->mask_buf); + kfree(d->main_status_buf); kfree(d->status_reg_buf); kfree(d->status_buf); + kfree(d->prev_status_buf); + if (d->config_buf) { + for (i = 0; i < d->chip->num_config_bases; i++) + kfree(d->config_buf[i]); + kfree(d->config_buf); + } + mutex_destroy(&d->lock); + lockdep_unregister_key(&d->lock_key); kfree(d); } EXPORT_SYMBOL_GPL(regmap_del_irq_chip); +static void devm_regmap_irq_chip_release(struct device *dev, void *res) +{ + struct regmap_irq_chip_data *d = *(struct regmap_irq_chip_data **)res; + + regmap_del_irq_chip(d->irq, d); +} + +static int devm_regmap_irq_chip_match(struct device *dev, void *res, void *data) + +{ + struct regmap_irq_chip_data **r = res; + + if (!r || !*r) { + WARN_ON(!r || !*r); + return 0; + } + return *r == data; +} + /** - * regmap_irq_chip_get_base(): Retrieve interrupt base for a regmap IRQ chip + * devm_regmap_add_irq_chip_fwnode() - Resource managed regmap_add_irq_chip_fwnode() * - * Useful for drivers to request their own IRQs. + * @dev: The device pointer on which irq_chip belongs to. + * @fwnode: The firmware node where the IRQ domain should be added to. + * @map: The regmap for the device. + * @irq: The IRQ the device uses to signal interrupts + * @irq_flags: The IRQF_ flags to use for the primary interrupt. + * @irq_base: Allocate at specific IRQ number if irq_base > 0. + * @chip: Configuration for the interrupt controller. + * @data: Runtime data structure for the controller, allocated on success * - * @data: regmap_irq controller to operate on. + * Returns 0 on success or an errno on failure. + * + * The ®map_irq_chip_data will be automatically released when the device is + * unbound. + */ +int devm_regmap_add_irq_chip_fwnode(struct device *dev, + struct fwnode_handle *fwnode, + struct regmap *map, int irq, + int irq_flags, int irq_base, + const struct regmap_irq_chip *chip, + struct regmap_irq_chip_data **data) +{ + struct regmap_irq_chip_data **ptr, *d; + int ret; + + ptr = devres_alloc(devm_regmap_irq_chip_release, sizeof(*ptr), + GFP_KERNEL); + if (!ptr) + return -ENOMEM; + + ret = regmap_add_irq_chip_fwnode(fwnode, map, irq, irq_flags, irq_base, + chip, &d); + if (ret < 0) { + devres_free(ptr); + return ret; + } + + *ptr = d; + devres_add(dev, ptr); + *data = d; + return 0; +} +EXPORT_SYMBOL_GPL(devm_regmap_add_irq_chip_fwnode); + +/** + * devm_regmap_add_irq_chip() - Resource managed regmap_add_irq_chip() + * + * @dev: The device pointer on which irq_chip belongs to. + * @map: The regmap for the device. + * @irq: The IRQ the device uses to signal interrupts + * @irq_flags: The IRQF_ flags to use for the primary interrupt. + * @irq_base: Allocate at specific IRQ number if irq_base > 0. + * @chip: Configuration for the interrupt controller. + * @data: Runtime data structure for the controller, allocated on success + * + * Returns 0 on success or an errno on failure. + * + * The ®map_irq_chip_data will be automatically released when the device is + * unbound. + */ +int devm_regmap_add_irq_chip(struct device *dev, struct regmap *map, int irq, + int irq_flags, int irq_base, + const struct regmap_irq_chip *chip, + struct regmap_irq_chip_data **data) +{ + return devm_regmap_add_irq_chip_fwnode(dev, dev_fwnode(map->dev), map, + irq, irq_flags, irq_base, chip, + data); +} +EXPORT_SYMBOL_GPL(devm_regmap_add_irq_chip); + +/** + * devm_regmap_del_irq_chip() - Resource managed regmap_del_irq_chip() + * + * @dev: Device for which the resource was allocated. + * @irq: Primary IRQ for the device. + * @data: ®map_irq_chip_data allocated by regmap_add_irq_chip(). + * + * A resource managed version of regmap_del_irq_chip(). + */ +void devm_regmap_del_irq_chip(struct device *dev, int irq, + struct regmap_irq_chip_data *data) +{ + int rc; + + WARN_ON(irq != data->irq); + rc = devres_release(dev, devm_regmap_irq_chip_release, + devm_regmap_irq_chip_match, data); + + if (rc != 0) + WARN_ON(rc); +} +EXPORT_SYMBOL_GPL(devm_regmap_del_irq_chip); + +/** + * regmap_irq_chip_get_base() - Retrieve interrupt base for a regmap IRQ chip + * + * @data: regmap irq controller to operate on. + * + * Useful for drivers to request their own IRQs. */ int regmap_irq_chip_get_base(struct regmap_irq_chip_data *data) { @@ -517,12 +1173,12 @@ int regmap_irq_chip_get_base(struct regmap_irq_chip_data *data) EXPORT_SYMBOL_GPL(regmap_irq_chip_get_base); /** - * regmap_irq_get_virq(): Map an interrupt on a chip to a virtual IRQ + * regmap_irq_get_virq() - Map an interrupt on a chip to a virtual IRQ * - * Useful for drivers to request their own IRQs. + * @data: regmap irq controller to operate on. + * @irq: index of the interrupt requested in the chip IRQs. * - * @data: regmap_irq controller to operate on. - * @irq: index of the interrupt requested in the chip IRQs + * Useful for drivers to request their own IRQs. */ int regmap_irq_get_virq(struct regmap_irq_chip_data *data, int irq) { @@ -535,14 +1191,14 @@ int regmap_irq_get_virq(struct regmap_irq_chip_data *data, int irq) EXPORT_SYMBOL_GPL(regmap_irq_get_virq); /** - * regmap_irq_get_domain(): Retrieve the irq_domain for the chip + * regmap_irq_get_domain() - Retrieve the irq_domain for the chip + * + * @data: regmap_irq controller to operate on. * * Useful for drivers to request their own IRQs and for integration * with subsystems. For ease of integration NULL is accepted as a * domain, allowing devices to just call this even if no domain is * allocated. - * - * @data: regmap_irq controller to operate on. */ struct irq_domain *regmap_irq_get_domain(struct regmap_irq_chip_data *data) { diff --git a/drivers/base/regmap/regmap-kunit.c b/drivers/base/regmap/regmap-kunit.c new file mode 100644 index 000000000000..f6fc5ed016da --- /dev/null +++ b/drivers/base/regmap/regmap-kunit.c @@ -0,0 +1,2131 @@ +// SPDX-License-Identifier: GPL-2.0 +// +// regmap KUnit tests +// +// Copyright 2023 Arm Ltd + +#include <kunit/device.h> +#include <kunit/resource.h> +#include <kunit/test.h> +#include "internal.h" + +#define BLOCK_TEST_SIZE 12 + +KUNIT_DEFINE_ACTION_WRAPPER(regmap_exit_action, regmap_exit, struct regmap *); + +struct regmap_test_priv { + struct device *dev; +}; + +struct regmap_test_param { + enum regcache_type cache; + enum regmap_endian val_endian; + + unsigned int from_reg; + bool fast_io; +}; + +static void get_changed_bytes(void *orig, void *new, size_t size) +{ + char *o = orig; + char *n = new; + int i; + + get_random_bytes(new, size); + + /* + * This could be nicer and more efficient but we shouldn't + * super care. + */ + for (i = 0; i < size; i++) + while (n[i] == o[i]) + get_random_bytes(&n[i], 1); +} + +static const struct regmap_config test_regmap_config = { + .reg_stride = 1, + .val_bits = sizeof(unsigned int) * 8, +}; + +static const char *regcache_type_name(enum regcache_type type) +{ + switch (type) { + case REGCACHE_NONE: + return "none"; + case REGCACHE_FLAT: + return "flat"; + case REGCACHE_FLAT_S: + return "flat-sparse"; + case REGCACHE_RBTREE: + return "rbtree"; + case REGCACHE_MAPLE: + return "maple"; + default: + return NULL; + } +} + +static const char *regmap_endian_name(enum regmap_endian endian) +{ + switch (endian) { + case REGMAP_ENDIAN_BIG: + return "big"; + case REGMAP_ENDIAN_LITTLE: + return "little"; + case REGMAP_ENDIAN_DEFAULT: + return "default"; + case REGMAP_ENDIAN_NATIVE: + return "native"; + default: + return NULL; + } +} + +static void param_to_desc(const struct regmap_test_param *param, char *desc) +{ + snprintf(desc, KUNIT_PARAM_DESC_SIZE, "%s-%s%s @%#x", + regcache_type_name(param->cache), + regmap_endian_name(param->val_endian), + param->fast_io ? " fast I/O" : "", + param->from_reg); +} + +static const struct regmap_test_param regcache_types_list[] = { + { .cache = REGCACHE_NONE }, + { .cache = REGCACHE_NONE, .fast_io = true }, + { .cache = REGCACHE_FLAT }, + { .cache = REGCACHE_FLAT, .fast_io = true }, + { .cache = REGCACHE_FLAT_S }, + { .cache = REGCACHE_FLAT_S, .fast_io = true }, + { .cache = REGCACHE_RBTREE }, + { .cache = REGCACHE_RBTREE, .fast_io = true }, + { .cache = REGCACHE_MAPLE }, + { .cache = REGCACHE_MAPLE, .fast_io = true }, +}; + +KUNIT_ARRAY_PARAM(regcache_types, regcache_types_list, param_to_desc); + +static const struct regmap_test_param real_cache_types_only_list[] = { + { .cache = REGCACHE_FLAT }, + { .cache = REGCACHE_FLAT, .fast_io = true }, + { .cache = REGCACHE_FLAT_S }, + { .cache = REGCACHE_FLAT_S, .fast_io = true }, + { .cache = REGCACHE_RBTREE }, + { .cache = REGCACHE_RBTREE, .fast_io = true }, + { .cache = REGCACHE_MAPLE }, + { .cache = REGCACHE_MAPLE, .fast_io = true }, +}; + +KUNIT_ARRAY_PARAM(real_cache_types_only, real_cache_types_only_list, param_to_desc); + +static const struct regmap_test_param real_cache_types_list[] = { + { .cache = REGCACHE_FLAT, .from_reg = 0 }, + { .cache = REGCACHE_FLAT, .from_reg = 0, .fast_io = true }, + { .cache = REGCACHE_FLAT, .from_reg = 0x2001 }, + { .cache = REGCACHE_FLAT, .from_reg = 0x2002 }, + { .cache = REGCACHE_FLAT, .from_reg = 0x2003 }, + { .cache = REGCACHE_FLAT, .from_reg = 0x2004 }, + { .cache = REGCACHE_FLAT_S, .from_reg = 0 }, + { .cache = REGCACHE_FLAT_S, .from_reg = 0, .fast_io = true }, + { .cache = REGCACHE_FLAT_S, .from_reg = 0x2001 }, + { .cache = REGCACHE_FLAT_S, .from_reg = 0x2002 }, + { .cache = REGCACHE_FLAT_S, .from_reg = 0x2003 }, + { .cache = REGCACHE_FLAT_S, .from_reg = 0x2004 }, + { .cache = REGCACHE_RBTREE, .from_reg = 0 }, + { .cache = REGCACHE_RBTREE, .from_reg = 0, .fast_io = true }, + { .cache = REGCACHE_RBTREE, .from_reg = 0x2001 }, + { .cache = REGCACHE_RBTREE, .from_reg = 0x2002 }, + { .cache = REGCACHE_RBTREE, .from_reg = 0x2003 }, + { .cache = REGCACHE_RBTREE, .from_reg = 0x2004 }, + { .cache = REGCACHE_MAPLE, .from_reg = 0 }, + { .cache = REGCACHE_MAPLE, .from_reg = 0, .fast_io = true }, + { .cache = REGCACHE_MAPLE, .from_reg = 0x2001 }, + { .cache = REGCACHE_MAPLE, .from_reg = 0x2002 }, + { .cache = REGCACHE_MAPLE, .from_reg = 0x2003 }, + { .cache = REGCACHE_MAPLE, .from_reg = 0x2004 }, +}; + +KUNIT_ARRAY_PARAM(real_cache_types, real_cache_types_list, param_to_desc); + +static const struct regmap_test_param sparse_cache_types_list[] = { + { .cache = REGCACHE_FLAT_S, .from_reg = 0 }, + { .cache = REGCACHE_FLAT_S, .from_reg = 0, .fast_io = true }, + { .cache = REGCACHE_FLAT_S, .from_reg = 0x2001 }, + { .cache = REGCACHE_FLAT_S, .from_reg = 0x2002 }, + { .cache = REGCACHE_FLAT_S, .from_reg = 0x2003 }, + { .cache = REGCACHE_FLAT_S, .from_reg = 0x2004 }, + { .cache = REGCACHE_RBTREE, .from_reg = 0 }, + { .cache = REGCACHE_RBTREE, .from_reg = 0, .fast_io = true }, + { .cache = REGCACHE_RBTREE, .from_reg = 0x2001 }, + { .cache = REGCACHE_RBTREE, .from_reg = 0x2002 }, + { .cache = REGCACHE_RBTREE, .from_reg = 0x2003 }, + { .cache = REGCACHE_RBTREE, .from_reg = 0x2004 }, + { .cache = REGCACHE_MAPLE, .from_reg = 0 }, + { .cache = REGCACHE_MAPLE, .from_reg = 0, .fast_io = true }, + { .cache = REGCACHE_MAPLE, .from_reg = 0x2001 }, + { .cache = REGCACHE_MAPLE, .from_reg = 0x2002 }, + { .cache = REGCACHE_MAPLE, .from_reg = 0x2003 }, + { .cache = REGCACHE_MAPLE, .from_reg = 0x2004 }, +}; + +KUNIT_ARRAY_PARAM(sparse_cache_types, sparse_cache_types_list, param_to_desc); + +static struct regmap *gen_regmap(struct kunit *test, + struct regmap_config *config, + struct regmap_ram_data **data) +{ + const struct regmap_test_param *param = test->param_value; + struct regmap_test_priv *priv = test->priv; + unsigned int *buf; + struct regmap *ret = ERR_PTR(-ENOMEM); + size_t size; + int i, error; + struct reg_default *defaults; + + config->cache_type = param->cache; + config->fast_io = param->fast_io; + + if (config->max_register == 0) { + config->max_register = param->from_reg; + if (config->num_reg_defaults) + config->max_register += (config->num_reg_defaults - 1) * + config->reg_stride; + else + config->max_register += (BLOCK_TEST_SIZE * config->reg_stride); + } + + size = array_size(config->max_register + 1, sizeof(*buf)); + buf = kmalloc(size, GFP_KERNEL); + if (!buf) + return ERR_PTR(-ENOMEM); + + get_random_bytes(buf, size); + + *data = kzalloc(sizeof(**data), GFP_KERNEL); + if (!(*data)) + goto out_free; + (*data)->vals = buf; + + if (config->num_reg_defaults) { + defaults = kunit_kcalloc(test, + config->num_reg_defaults, + sizeof(struct reg_default), + GFP_KERNEL); + if (!defaults) + goto out_free; + + config->reg_defaults = defaults; + + for (i = 0; i < config->num_reg_defaults; i++) { + defaults[i].reg = param->from_reg + (i * config->reg_stride); + defaults[i].def = buf[param->from_reg + (i * config->reg_stride)]; + } + } + + ret = regmap_init_ram(priv->dev, config, *data); + if (IS_ERR(ret)) + goto out_free; + + /* This calls regmap_exit() on failure, which frees buf and *data */ + error = kunit_add_action_or_reset(test, regmap_exit_action, ret); + if (error) + ret = ERR_PTR(error); + + return ret; + +out_free: + kfree(buf); + kfree(*data); + + return ret; +} + +static bool reg_5_false(struct device *dev, unsigned int reg) +{ + struct kunit *test = dev_get_drvdata(dev); + const struct regmap_test_param *param = test->param_value; + + return reg != (param->from_reg + 5); +} + +static void basic_read_write(struct kunit *test) +{ + struct regmap *map; + struct regmap_config config; + struct regmap_ram_data *data; + unsigned int val, rval; + + config = test_regmap_config; + + map = gen_regmap(test, &config, &data); + KUNIT_ASSERT_FALSE(test, IS_ERR(map)); + if (IS_ERR(map)) + return; + + get_random_bytes(&val, sizeof(val)); + + /* If we write a value to a register we can read it back */ + KUNIT_EXPECT_EQ(test, 0, regmap_write(map, 0, val)); + KUNIT_EXPECT_EQ(test, 0, regmap_read(map, 0, &rval)); + KUNIT_EXPECT_EQ(test, val, rval); + + /* If using a cache the cache satisfied the read */ + KUNIT_EXPECT_EQ(test, config.cache_type == REGCACHE_NONE, data->read[0]); +} + +static void bulk_write(struct kunit *test) +{ + struct regmap *map; + struct regmap_config config; + struct regmap_ram_data *data; + unsigned int val[BLOCK_TEST_SIZE], rval[BLOCK_TEST_SIZE]; + int i; + + config = test_regmap_config; + + map = gen_regmap(test, &config, &data); + KUNIT_ASSERT_FALSE(test, IS_ERR(map)); + if (IS_ERR(map)) + return; + + get_random_bytes(&val, sizeof(val)); + + /* + * Data written via the bulk API can be read back with single + * reads. + */ + KUNIT_EXPECT_EQ(test, 0, regmap_bulk_write(map, 0, val, + BLOCK_TEST_SIZE)); + for (i = 0; i < BLOCK_TEST_SIZE; i++) + KUNIT_EXPECT_EQ(test, 0, regmap_read(map, i, &rval[i])); + + KUNIT_EXPECT_MEMEQ(test, val, rval, sizeof(val)); + + /* If using a cache the cache satisfied the read */ + for (i = 0; i < BLOCK_TEST_SIZE; i++) + KUNIT_EXPECT_EQ(test, config.cache_type == REGCACHE_NONE, data->read[i]); +} + +static void bulk_read(struct kunit *test) +{ + struct regmap *map; + struct regmap_config config; + struct regmap_ram_data *data; + unsigned int val[BLOCK_TEST_SIZE], rval[BLOCK_TEST_SIZE]; + int i; + + config = test_regmap_config; + + map = gen_regmap(test, &config, &data); + KUNIT_ASSERT_FALSE(test, IS_ERR(map)); + if (IS_ERR(map)) + return; + + get_random_bytes(&val, sizeof(val)); + + /* Data written as single writes can be read via the bulk API */ + for (i = 0; i < BLOCK_TEST_SIZE; i++) + KUNIT_EXPECT_EQ(test, 0, regmap_write(map, i, val[i])); + KUNIT_EXPECT_EQ(test, 0, regmap_bulk_read(map, 0, rval, + BLOCK_TEST_SIZE)); + KUNIT_EXPECT_MEMEQ(test, val, rval, sizeof(val)); + + /* If using a cache the cache satisfied the read */ + for (i = 0; i < BLOCK_TEST_SIZE; i++) + KUNIT_EXPECT_EQ(test, config.cache_type == REGCACHE_NONE, data->read[i]); +} + +static void multi_write(struct kunit *test) +{ + struct regmap *map; + struct regmap_config config; + struct regmap_ram_data *data; + struct reg_sequence sequence[BLOCK_TEST_SIZE]; + unsigned int val[BLOCK_TEST_SIZE], rval[BLOCK_TEST_SIZE]; + int i; + + config = test_regmap_config; + + map = gen_regmap(test, &config, &data); + KUNIT_ASSERT_FALSE(test, IS_ERR(map)); + if (IS_ERR(map)) + return; + + get_random_bytes(&val, sizeof(val)); + + /* + * Data written via the multi API can be read back with single + * reads. + */ + for (i = 0; i < BLOCK_TEST_SIZE; i++) { + sequence[i].reg = i; + sequence[i].def = val[i]; + sequence[i].delay_us = 0; + } + KUNIT_EXPECT_EQ(test, 0, + regmap_multi_reg_write(map, sequence, BLOCK_TEST_SIZE)); + for (i = 0; i < BLOCK_TEST_SIZE; i++) + KUNIT_EXPECT_EQ(test, 0, regmap_read(map, i, &rval[i])); + + KUNIT_EXPECT_MEMEQ(test, val, rval, sizeof(val)); + + /* If using a cache the cache satisfied the read */ + for (i = 0; i < BLOCK_TEST_SIZE; i++) + KUNIT_EXPECT_EQ(test, config.cache_type == REGCACHE_NONE, data->read[i]); +} + +static void multi_read(struct kunit *test) +{ + struct regmap *map; + struct regmap_config config; + struct regmap_ram_data *data; + unsigned int regs[BLOCK_TEST_SIZE]; + unsigned int val[BLOCK_TEST_SIZE], rval[BLOCK_TEST_SIZE]; + int i; + + config = test_regmap_config; + + map = gen_regmap(test, &config, &data); + KUNIT_ASSERT_FALSE(test, IS_ERR(map)); + if (IS_ERR(map)) + return; + + get_random_bytes(&val, sizeof(val)); + + /* Data written as single writes can be read via the multi API */ + for (i = 0; i < BLOCK_TEST_SIZE; i++) { + regs[i] = i; + KUNIT_EXPECT_EQ(test, 0, regmap_write(map, i, val[i])); + } + KUNIT_EXPECT_EQ(test, 0, + regmap_multi_reg_read(map, regs, rval, BLOCK_TEST_SIZE)); + KUNIT_EXPECT_MEMEQ(test, val, rval, sizeof(val)); + + /* If using a cache the cache satisfied the read */ + for (i = 0; i < BLOCK_TEST_SIZE; i++) + KUNIT_EXPECT_EQ(test, config.cache_type == REGCACHE_NONE, data->read[i]); +} + +static void read_bypassed(struct kunit *test) +{ + const struct regmap_test_param *param = test->param_value; + struct regmap *map; + struct regmap_config config; + struct regmap_ram_data *data; + unsigned int val[BLOCK_TEST_SIZE], rval; + int i; + + config = test_regmap_config; + + map = gen_regmap(test, &config, &data); + KUNIT_ASSERT_FALSE(test, IS_ERR(map)); + if (IS_ERR(map)) + return; + + KUNIT_EXPECT_FALSE(test, map->cache_bypass); + + get_random_bytes(&val, sizeof(val)); + + /* Write some test values */ + KUNIT_EXPECT_EQ(test, 0, regmap_bulk_write(map, param->from_reg, val, ARRAY_SIZE(val))); + + regcache_cache_only(map, true); + + /* + * While in cache-only regmap_read_bypassed() should return the register + * value and leave the map in cache-only. + */ + for (i = 0; i < ARRAY_SIZE(val); i++) { + /* Put inverted bits in rval to prove we really read the value */ + rval = ~val[i]; + KUNIT_EXPECT_EQ(test, 0, regmap_read(map, param->from_reg + i, &rval)); + KUNIT_EXPECT_EQ(test, val[i], rval); + + rval = ~val[i]; + KUNIT_EXPECT_EQ(test, 0, regmap_read_bypassed(map, param->from_reg + i, &rval)); + KUNIT_EXPECT_EQ(test, val[i], rval); + KUNIT_EXPECT_TRUE(test, map->cache_only); + KUNIT_EXPECT_FALSE(test, map->cache_bypass); + } + + /* + * Change the underlying register values to prove it is returning + * real values not cached values. + */ + for (i = 0; i < ARRAY_SIZE(val); i++) { + val[i] = ~val[i]; + data->vals[param->from_reg + i] = val[i]; + } + + for (i = 0; i < ARRAY_SIZE(val); i++) { + rval = ~val[i]; + KUNIT_EXPECT_EQ(test, 0, regmap_read(map, param->from_reg + i, &rval)); + KUNIT_EXPECT_NE(test, val[i], rval); + + rval = ~val[i]; + KUNIT_EXPECT_EQ(test, 0, regmap_read_bypassed(map, param->from_reg + i, &rval)); + KUNIT_EXPECT_EQ(test, val[i], rval); + KUNIT_EXPECT_TRUE(test, map->cache_only); + KUNIT_EXPECT_FALSE(test, map->cache_bypass); + } +} + +static void read_bypassed_volatile(struct kunit *test) +{ + const struct regmap_test_param *param = test->param_value; + struct regmap *map; + struct regmap_config config; + struct regmap_ram_data *data; + unsigned int val[BLOCK_TEST_SIZE], rval; + int i; + + config = test_regmap_config; + /* All registers except #5 volatile */ + config.volatile_reg = reg_5_false; + + map = gen_regmap(test, &config, &data); + KUNIT_ASSERT_FALSE(test, IS_ERR(map)); + if (IS_ERR(map)) + return; + + KUNIT_EXPECT_FALSE(test, map->cache_bypass); + + get_random_bytes(&val, sizeof(val)); + + /* Write some test values */ + KUNIT_EXPECT_EQ(test, 0, regmap_bulk_write(map, param->from_reg, val, ARRAY_SIZE(val))); + + regcache_cache_only(map, true); + + /* + * While in cache-only regmap_read_bypassed() should return the register + * value and leave the map in cache-only. + */ + for (i = 0; i < ARRAY_SIZE(val); i++) { + /* Register #5 is non-volatile so should read from cache */ + KUNIT_EXPECT_EQ(test, (i == 5) ? 0 : -EBUSY, + regmap_read(map, param->from_reg + i, &rval)); + + /* Put inverted bits in rval to prove we really read the value */ + rval = ~val[i]; + KUNIT_EXPECT_EQ(test, 0, regmap_read_bypassed(map, param->from_reg + i, &rval)); + KUNIT_EXPECT_EQ(test, val[i], rval); + KUNIT_EXPECT_TRUE(test, map->cache_only); + KUNIT_EXPECT_FALSE(test, map->cache_bypass); + } + + /* + * Change the underlying register values to prove it is returning + * real values not cached values. + */ + for (i = 0; i < ARRAY_SIZE(val); i++) { + val[i] = ~val[i]; + data->vals[param->from_reg + i] = val[i]; + } + + for (i = 0; i < ARRAY_SIZE(val); i++) { + if (i == 5) + continue; + + rval = ~val[i]; + KUNIT_EXPECT_EQ(test, 0, regmap_read_bypassed(map, param->from_reg + i, &rval)); + KUNIT_EXPECT_EQ(test, val[i], rval); + KUNIT_EXPECT_TRUE(test, map->cache_only); + KUNIT_EXPECT_FALSE(test, map->cache_bypass); + } +} + +static void write_readonly(struct kunit *test) +{ + struct regmap *map; + struct regmap_config config; + struct regmap_ram_data *data; + unsigned int val; + int i; + + config = test_regmap_config; + config.num_reg_defaults = BLOCK_TEST_SIZE; + config.writeable_reg = reg_5_false; + + map = gen_regmap(test, &config, &data); + KUNIT_ASSERT_FALSE(test, IS_ERR(map)); + if (IS_ERR(map)) + return; + + get_random_bytes(&val, sizeof(val)); + + for (i = 0; i < BLOCK_TEST_SIZE; i++) + data->written[i] = false; + + /* Change the value of all registers, readonly should fail */ + for (i = 0; i < BLOCK_TEST_SIZE; i++) + KUNIT_EXPECT_EQ(test, i != 5, regmap_write(map, i, val) == 0); + + /* Did that match what we see on the device? */ + for (i = 0; i < BLOCK_TEST_SIZE; i++) + KUNIT_EXPECT_EQ(test, i != 5, data->written[i]); +} + +static void read_writeonly(struct kunit *test) +{ + struct regmap *map; + struct regmap_config config; + struct regmap_ram_data *data; + unsigned int val; + int i; + + config = test_regmap_config; + config.readable_reg = reg_5_false; + + map = gen_regmap(test, &config, &data); + KUNIT_ASSERT_FALSE(test, IS_ERR(map)); + if (IS_ERR(map)) + return; + + for (i = 0; i < BLOCK_TEST_SIZE; i++) + data->read[i] = false; + + /* + * Try to read all the registers, the writeonly one should + * fail if we aren't using the flat cache. + */ + for (i = 0; i < BLOCK_TEST_SIZE; i++) { + if (config.cache_type != REGCACHE_FLAT) { + KUNIT_EXPECT_EQ(test, i != 5, + regmap_read(map, i, &val) == 0); + } else { + KUNIT_EXPECT_EQ(test, 0, regmap_read(map, i, &val)); + } + } + + /* Did we trigger a hardware access? */ + KUNIT_EXPECT_FALSE(test, data->read[5]); +} + +static void reg_defaults(struct kunit *test) +{ + struct regmap *map; + struct regmap_config config; + struct regmap_ram_data *data; + unsigned int rval[BLOCK_TEST_SIZE]; + int i; + + config = test_regmap_config; + config.num_reg_defaults = BLOCK_TEST_SIZE; + + map = gen_regmap(test, &config, &data); + KUNIT_ASSERT_FALSE(test, IS_ERR(map)); + if (IS_ERR(map)) + return; + + /* Read back the expected default data */ + KUNIT_EXPECT_EQ(test, 0, regmap_bulk_read(map, 0, rval, + BLOCK_TEST_SIZE)); + KUNIT_EXPECT_MEMEQ(test, data->vals, rval, sizeof(rval)); + + /* The data should have been read from cache if there was one */ + for (i = 0; i < BLOCK_TEST_SIZE; i++) + KUNIT_EXPECT_EQ(test, config.cache_type == REGCACHE_NONE, data->read[i]); +} + +static void reg_defaults_read_dev(struct kunit *test) +{ + struct regmap *map; + struct regmap_config config; + struct regmap_ram_data *data; + unsigned int rval[BLOCK_TEST_SIZE]; + int i; + + config = test_regmap_config; + config.num_reg_defaults_raw = BLOCK_TEST_SIZE; + + map = gen_regmap(test, &config, &data); + KUNIT_ASSERT_FALSE(test, IS_ERR(map)); + if (IS_ERR(map)) + return; + + /* We should have read the cache defaults back from the map */ + for (i = 0; i < BLOCK_TEST_SIZE; i++) { + KUNIT_EXPECT_EQ(test, config.cache_type != REGCACHE_NONE, data->read[i]); + data->read[i] = false; + } + + /* Read back the expected default data */ + KUNIT_EXPECT_EQ(test, 0, regmap_bulk_read(map, 0, rval, + BLOCK_TEST_SIZE)); + KUNIT_EXPECT_MEMEQ(test, data->vals, rval, sizeof(rval)); + + /* The data should have been read from cache if there was one */ + for (i = 0; i < BLOCK_TEST_SIZE; i++) + KUNIT_EXPECT_EQ(test, config.cache_type == REGCACHE_NONE, data->read[i]); +} + +static void register_patch(struct kunit *test) +{ + struct regmap *map; + struct regmap_config config; + struct regmap_ram_data *data; + struct reg_sequence patch[2]; + unsigned int rval[BLOCK_TEST_SIZE]; + int i; + + /* We need defaults so readback works */ + config = test_regmap_config; + config.num_reg_defaults = BLOCK_TEST_SIZE; + + map = gen_regmap(test, &config, &data); + KUNIT_ASSERT_FALSE(test, IS_ERR(map)); + if (IS_ERR(map)) + return; + + /* Stash the original values */ + KUNIT_EXPECT_EQ(test, 0, regmap_bulk_read(map, 0, rval, + BLOCK_TEST_SIZE)); + + /* Patch a couple of values */ + patch[0].reg = 2; + patch[0].def = rval[2] + 1; + patch[0].delay_us = 0; + patch[1].reg = 5; + patch[1].def = rval[5] + 1; + patch[1].delay_us = 0; + KUNIT_EXPECT_EQ(test, 0, regmap_register_patch(map, patch, + ARRAY_SIZE(patch))); + + /* Only the patched registers are written */ + for (i = 0; i < BLOCK_TEST_SIZE; i++) { + switch (i) { + case 2: + case 5: + KUNIT_EXPECT_TRUE(test, data->written[i]); + KUNIT_EXPECT_EQ(test, data->vals[i], rval[i] + 1); + break; + default: + KUNIT_EXPECT_FALSE(test, data->written[i]); + KUNIT_EXPECT_EQ(test, data->vals[i], rval[i]); + break; + } + } +} + +static void stride(struct kunit *test) +{ + struct regmap *map; + struct regmap_config config; + struct regmap_ram_data *data; + unsigned int rval; + int i; + + config = test_regmap_config; + config.reg_stride = 2; + config.num_reg_defaults = BLOCK_TEST_SIZE / 2; + + /* + * Allow one extra register so that the read/written arrays + * are sized big enough to include an entry for the odd + * address past the final reg_default register. + */ + config.max_register = BLOCK_TEST_SIZE; + + map = gen_regmap(test, &config, &data); + KUNIT_ASSERT_FALSE(test, IS_ERR(map)); + if (IS_ERR(map)) + return; + + /* Only even addresses can be accessed, try both read and write */ + for (i = 0; i < BLOCK_TEST_SIZE; i++) { + data->read[i] = false; + data->written[i] = false; + + if (i % 2) { + KUNIT_EXPECT_NE(test, 0, regmap_read(map, i, &rval)); + KUNIT_EXPECT_NE(test, 0, regmap_write(map, i, rval)); + KUNIT_EXPECT_FALSE(test, data->read[i]); + KUNIT_EXPECT_FALSE(test, data->written[i]); + } else { + KUNIT_EXPECT_EQ(test, 0, regmap_read(map, i, &rval)); + KUNIT_EXPECT_EQ(test, data->vals[i], rval); + KUNIT_EXPECT_EQ(test, config.cache_type == REGCACHE_NONE, + data->read[i]); + + KUNIT_EXPECT_EQ(test, 0, regmap_write(map, i, rval)); + KUNIT_EXPECT_TRUE(test, data->written[i]); + } + } +} + +static const struct regmap_range_cfg test_range = { + .selector_reg = 1, + .selector_mask = 0xff, + + .window_start = 4, + .window_len = 10, + + .range_min = 20, + .range_max = 40, +}; + +static bool test_range_window_volatile(struct device *dev, unsigned int reg) +{ + if (reg >= test_range.window_start && + reg <= test_range.window_start + test_range.window_len) + return true; + + return false; +} + +static bool test_range_all_volatile(struct device *dev, unsigned int reg) +{ + if (test_range_window_volatile(dev, reg)) + return true; + + if (reg >= test_range.range_min && reg <= test_range.range_max) + return true; + + return false; +} + +static void basic_ranges(struct kunit *test) +{ + struct regmap *map; + struct regmap_config config; + struct regmap_ram_data *data; + unsigned int val; + int i; + + config = test_regmap_config; + config.volatile_reg = test_range_all_volatile; + config.ranges = &test_range; + config.num_ranges = 1; + config.max_register = test_range.range_max; + + map = gen_regmap(test, &config, &data); + KUNIT_ASSERT_FALSE(test, IS_ERR(map)); + if (IS_ERR(map)) + return; + + for (i = test_range.range_min; i < test_range.range_max; i++) { + data->read[i] = false; + data->written[i] = false; + } + + /* Reset the page to a non-zero value to trigger a change */ + KUNIT_EXPECT_EQ(test, 0, regmap_write(map, test_range.selector_reg, + test_range.range_max)); + + /* Check we set the page and use the window for writes */ + data->written[test_range.selector_reg] = false; + data->written[test_range.window_start] = false; + KUNIT_EXPECT_EQ(test, 0, regmap_write(map, test_range.range_min, 0)); + KUNIT_EXPECT_TRUE(test, data->written[test_range.selector_reg]); + KUNIT_EXPECT_TRUE(test, data->written[test_range.window_start]); + + data->written[test_range.selector_reg] = false; + data->written[test_range.window_start] = false; + KUNIT_EXPECT_EQ(test, 0, regmap_write(map, + test_range.range_min + + test_range.window_len, + 0)); + KUNIT_EXPECT_TRUE(test, data->written[test_range.selector_reg]); + KUNIT_EXPECT_TRUE(test, data->written[test_range.window_start]); + + /* Same for reads */ + data->written[test_range.selector_reg] = false; + data->read[test_range.window_start] = false; + KUNIT_EXPECT_EQ(test, 0, regmap_read(map, test_range.range_min, &val)); + KUNIT_EXPECT_TRUE(test, data->written[test_range.selector_reg]); + KUNIT_EXPECT_TRUE(test, data->read[test_range.window_start]); + + data->written[test_range.selector_reg] = false; + data->read[test_range.window_start] = false; + KUNIT_EXPECT_EQ(test, 0, regmap_read(map, + test_range.range_min + + test_range.window_len, + &val)); + KUNIT_EXPECT_TRUE(test, data->written[test_range.selector_reg]); + KUNIT_EXPECT_TRUE(test, data->read[test_range.window_start]); + + /* No physical access triggered in the virtual range */ + for (i = test_range.range_min; i < test_range.range_max; i++) { + KUNIT_EXPECT_FALSE(test, data->read[i]); + KUNIT_EXPECT_FALSE(test, data->written[i]); + } +} + +/* Try to stress dynamic creation of cache data structures */ +static void stress_insert(struct kunit *test) +{ + struct regmap *map; + struct regmap_config config; + struct regmap_ram_data *data; + unsigned int rval, *vals; + size_t buf_sz; + int i; + + config = test_regmap_config; + config.max_register = 300; + + map = gen_regmap(test, &config, &data); + KUNIT_ASSERT_FALSE(test, IS_ERR(map)); + if (IS_ERR(map)) + return; + + buf_sz = array_size(sizeof(*vals), config.max_register); + vals = kunit_kmalloc(test, buf_sz, GFP_KERNEL); + KUNIT_ASSERT_FALSE(test, vals == NULL); + + get_random_bytes(vals, buf_sz); + + /* Write data into the map/cache in ever decreasing strides */ + for (i = 0; i < config.max_register; i += 100) + KUNIT_EXPECT_EQ(test, 0, regmap_write(map, i, vals[i])); + for (i = 0; i < config.max_register; i += 50) + KUNIT_EXPECT_EQ(test, 0, regmap_write(map, i, vals[i])); + for (i = 0; i < config.max_register; i += 25) + KUNIT_EXPECT_EQ(test, 0, regmap_write(map, i, vals[i])); + for (i = 0; i < config.max_register; i += 10) + KUNIT_EXPECT_EQ(test, 0, regmap_write(map, i, vals[i])); + for (i = 0; i < config.max_register; i += 5) + KUNIT_EXPECT_EQ(test, 0, regmap_write(map, i, vals[i])); + for (i = 0; i < config.max_register; i += 3) + KUNIT_EXPECT_EQ(test, 0, regmap_write(map, i, vals[i])); + for (i = 0; i < config.max_register; i += 2) + KUNIT_EXPECT_EQ(test, 0, regmap_write(map, i, vals[i])); + for (i = 0; i < config.max_register; i++) + KUNIT_EXPECT_EQ(test, 0, regmap_write(map, i, vals[i])); + + /* Do reads from the cache (if there is one) match? */ + for (i = 0; i < config.max_register; i ++) { + KUNIT_EXPECT_EQ(test, 0, regmap_read(map, i, &rval)); + KUNIT_EXPECT_EQ(test, rval, vals[i]); + KUNIT_EXPECT_EQ(test, config.cache_type == REGCACHE_NONE, data->read[i]); + } +} + +static void cache_bypass(struct kunit *test) +{ + const struct regmap_test_param *param = test->param_value; + struct regmap *map; + struct regmap_config config; + struct regmap_ram_data *data; + unsigned int val, rval; + + config = test_regmap_config; + + map = gen_regmap(test, &config, &data); + KUNIT_ASSERT_FALSE(test, IS_ERR(map)); + if (IS_ERR(map)) + return; + + get_random_bytes(&val, sizeof(val)); + + /* Ensure the cache has a value in it */ + KUNIT_EXPECT_EQ(test, 0, regmap_write(map, param->from_reg, val)); + + /* Bypass then write a different value */ + regcache_cache_bypass(map, true); + KUNIT_EXPECT_EQ(test, 0, regmap_write(map, param->from_reg, val + 1)); + + /* Read the bypassed value */ + KUNIT_EXPECT_EQ(test, 0, regmap_read(map, param->from_reg, &rval)); + KUNIT_EXPECT_EQ(test, val + 1, rval); + KUNIT_EXPECT_EQ(test, data->vals[param->from_reg], rval); + + /* Disable bypass, the cache should still return the original value */ + regcache_cache_bypass(map, false); + KUNIT_EXPECT_EQ(test, 0, regmap_read(map, param->from_reg, &rval)); + KUNIT_EXPECT_EQ(test, val, rval); +} + +static void cache_sync_marked_dirty(struct kunit *test) +{ + const struct regmap_test_param *param = test->param_value; + struct regmap *map; + struct regmap_config config; + struct regmap_ram_data *data; + unsigned int val[BLOCK_TEST_SIZE]; + int i; + + config = test_regmap_config; + + map = gen_regmap(test, &config, &data); + KUNIT_ASSERT_FALSE(test, IS_ERR(map)); + if (IS_ERR(map)) + return; + + get_random_bytes(&val, sizeof(val)); + + /* Put some data into the cache */ + KUNIT_EXPECT_EQ(test, 0, regmap_bulk_write(map, param->from_reg, val, + BLOCK_TEST_SIZE)); + for (i = 0; i < BLOCK_TEST_SIZE; i++) + data->written[param->from_reg + i] = false; + + /* Trash the data on the device itself then resync */ + regcache_mark_dirty(map); + memset(data->vals, 0, sizeof(val)); + KUNIT_EXPECT_EQ(test, 0, regcache_sync(map)); + + /* Did we just write the correct data out? */ + KUNIT_EXPECT_MEMEQ(test, &data->vals[param->from_reg], val, sizeof(val)); + for (i = 0; i < BLOCK_TEST_SIZE; i++) + KUNIT_EXPECT_EQ(test, true, data->written[param->from_reg + i]); +} + +static void cache_sync_after_cache_only(struct kunit *test) +{ + const struct regmap_test_param *param = test->param_value; + struct regmap *map; + struct regmap_config config; + struct regmap_ram_data *data; + unsigned int val[BLOCK_TEST_SIZE]; + unsigned int val_mask; + int i; + + config = test_regmap_config; + + map = gen_regmap(test, &config, &data); + KUNIT_ASSERT_FALSE(test, IS_ERR(map)); + if (IS_ERR(map)) + return; + + val_mask = GENMASK(config.val_bits - 1, 0); + get_random_bytes(&val, sizeof(val)); + + /* Put some data into the cache */ + KUNIT_EXPECT_EQ(test, 0, regmap_bulk_write(map, param->from_reg, val, + BLOCK_TEST_SIZE)); + for (i = 0; i < BLOCK_TEST_SIZE; i++) + data->written[param->from_reg + i] = false; + + /* Set cache-only and change the values */ + regcache_cache_only(map, true); + for (i = 0; i < ARRAY_SIZE(val); ++i) + val[i] = ~val[i] & val_mask; + + KUNIT_EXPECT_EQ(test, 0, regmap_bulk_write(map, param->from_reg, val, + BLOCK_TEST_SIZE)); + for (i = 0; i < BLOCK_TEST_SIZE; i++) + KUNIT_EXPECT_FALSE(test, data->written[param->from_reg + i]); + + KUNIT_EXPECT_MEMNEQ(test, &data->vals[param->from_reg], val, sizeof(val)); + + /* Exit cache-only and sync the cache without marking hardware registers dirty */ + regcache_cache_only(map, false); + + KUNIT_EXPECT_EQ(test, 0, regcache_sync(map)); + + /* Did we just write the correct data out? */ + KUNIT_EXPECT_MEMEQ(test, &data->vals[param->from_reg], val, sizeof(val)); + for (i = 0; i < BLOCK_TEST_SIZE; i++) + KUNIT_EXPECT_TRUE(test, data->written[param->from_reg + i]); +} + +static void cache_sync_defaults_marked_dirty(struct kunit *test) +{ + const struct regmap_test_param *param = test->param_value; + struct regmap *map; + struct regmap_config config; + struct regmap_ram_data *data; + unsigned int val; + int i; + + config = test_regmap_config; + config.num_reg_defaults = BLOCK_TEST_SIZE; + + map = gen_regmap(test, &config, &data); + KUNIT_ASSERT_FALSE(test, IS_ERR(map)); + if (IS_ERR(map)) + return; + + get_random_bytes(&val, sizeof(val)); + + /* Change the value of one register */ + KUNIT_EXPECT_EQ(test, 0, regmap_write(map, param->from_reg + 2, val)); + + /* Resync */ + regcache_mark_dirty(map); + for (i = 0; i < BLOCK_TEST_SIZE; i++) + data->written[param->from_reg + i] = false; + KUNIT_EXPECT_EQ(test, 0, regcache_sync(map)); + + /* Did we just sync the one register we touched? */ + for (i = 0; i < BLOCK_TEST_SIZE; i++) + KUNIT_EXPECT_EQ(test, i == 2, data->written[param->from_reg + i]); + + /* Rewrite registers back to their defaults */ + for (i = 0; i < config.num_reg_defaults; ++i) + KUNIT_EXPECT_EQ(test, 0, regmap_write(map, config.reg_defaults[i].reg, + config.reg_defaults[i].def)); + + /* + * Resync after regcache_mark_dirty() should not write out registers + * that are at default value + */ + for (i = 0; i < BLOCK_TEST_SIZE; i++) + data->written[param->from_reg + i] = false; + regcache_mark_dirty(map); + KUNIT_EXPECT_EQ(test, 0, regcache_sync(map)); + for (i = 0; i < BLOCK_TEST_SIZE; i++) + KUNIT_EXPECT_FALSE(test, data->written[param->from_reg + i]); +} + +static void cache_sync_default_after_cache_only(struct kunit *test) +{ + const struct regmap_test_param *param = test->param_value; + struct regmap *map; + struct regmap_config config; + struct regmap_ram_data *data; + unsigned int orig_val; + int i; + + config = test_regmap_config; + config.num_reg_defaults = BLOCK_TEST_SIZE; + + map = gen_regmap(test, &config, &data); + KUNIT_ASSERT_FALSE(test, IS_ERR(map)); + if (IS_ERR(map)) + return; + + KUNIT_EXPECT_EQ(test, 0, regmap_read(map, param->from_reg + 2, &orig_val)); + + /* Enter cache-only and change the value of one register */ + regcache_cache_only(map, true); + KUNIT_EXPECT_EQ(test, 0, regmap_write(map, param->from_reg + 2, orig_val + 1)); + + /* Exit cache-only and resync, should write out the changed register */ + regcache_cache_only(map, false); + for (i = 0; i < BLOCK_TEST_SIZE; i++) + data->written[param->from_reg + i] = false; + KUNIT_EXPECT_EQ(test, 0, regcache_sync(map)); + + /* Was the register written out? */ + KUNIT_EXPECT_TRUE(test, data->written[param->from_reg + 2]); + KUNIT_EXPECT_EQ(test, data->vals[param->from_reg + 2], orig_val + 1); + + /* Enter cache-only and write register back to its default value */ + regcache_cache_only(map, true); + KUNIT_EXPECT_EQ(test, 0, regmap_write(map, param->from_reg + 2, orig_val)); + + /* Resync should write out the new value */ + regcache_cache_only(map, false); + for (i = 0; i < BLOCK_TEST_SIZE; i++) + data->written[param->from_reg + i] = false; + + KUNIT_EXPECT_EQ(test, 0, regcache_sync(map)); + KUNIT_EXPECT_TRUE(test, data->written[param->from_reg + 2]); + KUNIT_EXPECT_EQ(test, data->vals[param->from_reg + 2], orig_val); +} + +static void cache_sync_readonly(struct kunit *test) +{ + const struct regmap_test_param *param = test->param_value; + struct regmap *map; + struct regmap_config config; + struct regmap_ram_data *data; + unsigned int val; + int i; + + config = test_regmap_config; + config.writeable_reg = reg_5_false; + + map = gen_regmap(test, &config, &data); + KUNIT_ASSERT_FALSE(test, IS_ERR(map)); + if (IS_ERR(map)) + return; + + /* Read all registers to fill the cache */ + for (i = 0; i < BLOCK_TEST_SIZE; i++) + KUNIT_EXPECT_EQ(test, 0, regmap_read(map, param->from_reg + i, &val)); + + /* Change the value of all registers, readonly should fail */ + get_random_bytes(&val, sizeof(val)); + regcache_cache_only(map, true); + for (i = 0; i < BLOCK_TEST_SIZE; i++) + KUNIT_EXPECT_EQ(test, i != 5, regmap_write(map, param->from_reg + i, val) == 0); + regcache_cache_only(map, false); + + /* Resync */ + for (i = 0; i < BLOCK_TEST_SIZE; i++) + data->written[param->from_reg + i] = false; + KUNIT_EXPECT_EQ(test, 0, regcache_sync(map)); + + /* Did that match what we see on the device? */ + for (i = 0; i < BLOCK_TEST_SIZE; i++) + KUNIT_EXPECT_EQ(test, i != 5, data->written[param->from_reg + i]); +} + +static void cache_sync_patch(struct kunit *test) +{ + const struct regmap_test_param *param = test->param_value; + struct regmap *map; + struct regmap_config config; + struct regmap_ram_data *data; + struct reg_sequence patch[2]; + unsigned int rval[BLOCK_TEST_SIZE], val; + int i; + + /* We need defaults so readback works */ + config = test_regmap_config; + config.num_reg_defaults = BLOCK_TEST_SIZE; + + map = gen_regmap(test, &config, &data); + KUNIT_ASSERT_FALSE(test, IS_ERR(map)); + if (IS_ERR(map)) + return; + + /* Stash the original values */ + KUNIT_EXPECT_EQ(test, 0, regmap_bulk_read(map, param->from_reg, rval, + BLOCK_TEST_SIZE)); + + /* Patch a couple of values */ + patch[0].reg = param->from_reg + 2; + patch[0].def = rval[2] + 1; + patch[0].delay_us = 0; + patch[1].reg = param->from_reg + 5; + patch[1].def = rval[5] + 1; + patch[1].delay_us = 0; + KUNIT_EXPECT_EQ(test, 0, regmap_register_patch(map, patch, + ARRAY_SIZE(patch))); + + /* Sync the cache */ + regcache_mark_dirty(map); + for (i = 0; i < BLOCK_TEST_SIZE; i++) + data->written[param->from_reg + i] = false; + KUNIT_EXPECT_EQ(test, 0, regcache_sync(map)); + + /* The patch should be on the device but not in the cache */ + for (i = 0; i < BLOCK_TEST_SIZE; i++) { + KUNIT_EXPECT_EQ(test, 0, regmap_read(map, param->from_reg + i, &val)); + KUNIT_EXPECT_EQ(test, val, rval[i]); + + switch (i) { + case 2: + case 5: + KUNIT_EXPECT_EQ(test, true, data->written[param->from_reg + i]); + KUNIT_EXPECT_EQ(test, data->vals[param->from_reg + i], rval[i] + 1); + break; + default: + KUNIT_EXPECT_EQ(test, false, data->written[param->from_reg + i]); + KUNIT_EXPECT_EQ(test, data->vals[param->from_reg + i], rval[i]); + break; + } + } +} + +static void cache_drop(struct kunit *test) +{ + const struct regmap_test_param *param = test->param_value; + struct regmap *map; + struct regmap_config config; + struct regmap_ram_data *data; + unsigned int rval[BLOCK_TEST_SIZE]; + int i; + + config = test_regmap_config; + config.num_reg_defaults = BLOCK_TEST_SIZE; + + map = gen_regmap(test, &config, &data); + KUNIT_ASSERT_FALSE(test, IS_ERR(map)); + if (IS_ERR(map)) + return; + + /* Ensure the data is read from the cache */ + for (i = 0; i < BLOCK_TEST_SIZE; i++) + data->read[param->from_reg + i] = false; + KUNIT_EXPECT_EQ(test, 0, regmap_bulk_read(map, param->from_reg, rval, + BLOCK_TEST_SIZE)); + for (i = 0; i < BLOCK_TEST_SIZE; i++) { + KUNIT_EXPECT_FALSE(test, data->read[param->from_reg + i]); + data->read[param->from_reg + i] = false; + } + KUNIT_EXPECT_MEMEQ(test, &data->vals[param->from_reg], rval, sizeof(rval)); + + /* Drop some registers */ + KUNIT_EXPECT_EQ(test, 0, regcache_drop_region(map, param->from_reg + 3, + param->from_reg + 5)); + + /* Reread and check only the dropped registers hit the device. */ + KUNIT_EXPECT_EQ(test, 0, regmap_bulk_read(map, param->from_reg, rval, + BLOCK_TEST_SIZE)); + for (i = 0; i < BLOCK_TEST_SIZE; i++) + KUNIT_EXPECT_EQ(test, data->read[param->from_reg + i], i >= 3 && i <= 5); + KUNIT_EXPECT_MEMEQ(test, &data->vals[param->from_reg], rval, sizeof(rval)); +} + +static void cache_drop_with_non_contiguous_ranges(struct kunit *test) +{ + const struct regmap_test_param *param = test->param_value; + struct regmap *map; + struct regmap_config config; + struct regmap_ram_data *data; + unsigned int val[4][BLOCK_TEST_SIZE]; + unsigned int reg; + const int num_ranges = ARRAY_SIZE(val) * 2; + int rangeidx, i; + + static_assert(ARRAY_SIZE(val) == 4); + + config = test_regmap_config; + config.max_register = param->from_reg + (num_ranges * BLOCK_TEST_SIZE); + + map = gen_regmap(test, &config, &data); + KUNIT_ASSERT_FALSE(test, IS_ERR(map)); + if (IS_ERR(map)) + return; + + for (i = 0; i < config.max_register + 1; i++) + data->written[i] = false; + + /* Create non-contiguous cache blocks by writing every other range */ + get_random_bytes(&val, sizeof(val)); + for (rangeidx = 0; rangeidx < num_ranges; rangeidx += 2) { + reg = param->from_reg + (rangeidx * BLOCK_TEST_SIZE); + KUNIT_EXPECT_EQ(test, 0, regmap_bulk_write(map, reg, + &val[rangeidx / 2], + BLOCK_TEST_SIZE)); + KUNIT_EXPECT_MEMEQ(test, &data->vals[reg], + &val[rangeidx / 2], sizeof(val[rangeidx / 2])); + } + + /* Check that odd ranges weren't written */ + for (rangeidx = 1; rangeidx < num_ranges; rangeidx += 2) { + reg = param->from_reg + (rangeidx * BLOCK_TEST_SIZE); + for (i = 0; i < BLOCK_TEST_SIZE; i++) + KUNIT_EXPECT_FALSE(test, data->written[reg + i]); + } + + /* Drop range 2 */ + reg = param->from_reg + (2 * BLOCK_TEST_SIZE); + KUNIT_EXPECT_EQ(test, 0, regcache_drop_region(map, reg, reg + BLOCK_TEST_SIZE - 1)); + + /* Drop part of range 4 */ + reg = param->from_reg + (4 * BLOCK_TEST_SIZE); + KUNIT_EXPECT_EQ(test, 0, regcache_drop_region(map, reg + 3, reg + 5)); + + /* Mark dirty and reset mock registers to 0 */ + regcache_mark_dirty(map); + for (i = 0; i < config.max_register + 1; i++) { + data->vals[i] = 0; + data->written[i] = false; + } + + /* The registers that were dropped from range 4 should now remain at 0 */ + val[4 / 2][3] = 0; + val[4 / 2][4] = 0; + val[4 / 2][5] = 0; + + /* Sync and check that the expected register ranges were written */ + KUNIT_EXPECT_EQ(test, 0, regcache_sync(map)); + + /* Check that odd ranges weren't written */ + for (rangeidx = 1; rangeidx < num_ranges; rangeidx += 2) { + reg = param->from_reg + (rangeidx * BLOCK_TEST_SIZE); + for (i = 0; i < BLOCK_TEST_SIZE; i++) + KUNIT_EXPECT_FALSE(test, data->written[reg + i]); + } + + /* Check that even ranges (except 2 and 4) were written */ + for (rangeidx = 0; rangeidx < num_ranges; rangeidx += 2) { + if ((rangeidx == 2) || (rangeidx == 4)) + continue; + + reg = param->from_reg + (rangeidx * BLOCK_TEST_SIZE); + for (i = 0; i < BLOCK_TEST_SIZE; i++) + KUNIT_EXPECT_TRUE(test, data->written[reg + i]); + + KUNIT_EXPECT_MEMEQ(test, &data->vals[reg], + &val[rangeidx / 2], sizeof(val[rangeidx / 2])); + } + + /* Check that range 2 wasn't written */ + reg = param->from_reg + (2 * BLOCK_TEST_SIZE); + for (i = 0; i < BLOCK_TEST_SIZE; i++) + KUNIT_EXPECT_FALSE(test, data->written[reg + i]); + + /* Check that range 4 was partially written */ + reg = param->from_reg + (4 * BLOCK_TEST_SIZE); + for (i = 0; i < BLOCK_TEST_SIZE; i++) + KUNIT_EXPECT_EQ(test, data->written[reg + i], i < 3 || i > 5); + + KUNIT_EXPECT_MEMEQ(test, &data->vals[reg], &val[4 / 2], sizeof(val[4 / 2])); + + /* Nothing before param->from_reg should have been written */ + for (i = 0; i < param->from_reg; i++) + KUNIT_EXPECT_FALSE(test, data->written[i]); +} + +static void cache_drop_all_and_sync_marked_dirty(struct kunit *test) +{ + const struct regmap_test_param *param = test->param_value; + struct regmap *map; + struct regmap_config config; + struct regmap_ram_data *data; + unsigned int rval[BLOCK_TEST_SIZE]; + int i; + + config = test_regmap_config; + config.num_reg_defaults = BLOCK_TEST_SIZE; + + map = gen_regmap(test, &config, &data); + KUNIT_ASSERT_FALSE(test, IS_ERR(map)); + if (IS_ERR(map)) + return; + + /* Ensure the data is read from the cache */ + for (i = 0; i < BLOCK_TEST_SIZE; i++) + data->read[param->from_reg + i] = false; + KUNIT_EXPECT_EQ(test, 0, regmap_bulk_read(map, param->from_reg, rval, + BLOCK_TEST_SIZE)); + KUNIT_EXPECT_MEMEQ(test, &data->vals[param->from_reg], rval, sizeof(rval)); + + /* Change all values in cache from defaults */ + for (i = 0; i < BLOCK_TEST_SIZE; i++) + KUNIT_EXPECT_EQ(test, 0, regmap_write(map, param->from_reg + i, rval[i] + 1)); + + /* Drop all registers */ + KUNIT_EXPECT_EQ(test, 0, regcache_drop_region(map, 0, config.max_register)); + + /* Mark dirty and cache sync should not write anything. */ + regcache_mark_dirty(map); + for (i = 0; i < BLOCK_TEST_SIZE; i++) + data->written[param->from_reg + i] = false; + + KUNIT_EXPECT_EQ(test, 0, regcache_sync(map)); + for (i = 0; i <= config.max_register; i++) + KUNIT_EXPECT_FALSE(test, data->written[i]); +} + +static void cache_drop_all_and_sync_no_defaults(struct kunit *test) +{ + const struct regmap_test_param *param = test->param_value; + struct regmap *map; + struct regmap_config config; + struct regmap_ram_data *data; + unsigned int rval[BLOCK_TEST_SIZE]; + int i; + + config = test_regmap_config; + + map = gen_regmap(test, &config, &data); + KUNIT_ASSERT_FALSE(test, IS_ERR(map)); + if (IS_ERR(map)) + return; + + /* Ensure the data is read from the cache */ + for (i = 0; i < BLOCK_TEST_SIZE; i++) + data->read[param->from_reg + i] = false; + KUNIT_EXPECT_EQ(test, 0, regmap_bulk_read(map, param->from_reg, rval, + BLOCK_TEST_SIZE)); + KUNIT_EXPECT_MEMEQ(test, &data->vals[param->from_reg], rval, sizeof(rval)); + + /* Change all values in cache */ + for (i = 0; i < BLOCK_TEST_SIZE; i++) + KUNIT_EXPECT_EQ(test, 0, regmap_write(map, param->from_reg + i, rval[i] + 1)); + + /* Drop all registers */ + KUNIT_EXPECT_EQ(test, 0, regcache_drop_region(map, 0, config.max_register)); + + /* + * Sync cache without marking it dirty. All registers were dropped + * so the cache should not have any entries to write out. + */ + for (i = 0; i < BLOCK_TEST_SIZE; i++) + data->written[param->from_reg + i] = false; + + KUNIT_EXPECT_EQ(test, 0, regcache_sync(map)); + for (i = 0; i <= config.max_register; i++) + KUNIT_EXPECT_FALSE(test, data->written[i]); +} + +static void cache_drop_all_and_sync_has_defaults(struct kunit *test) +{ + const struct regmap_test_param *param = test->param_value; + struct regmap *map; + struct regmap_config config; + struct regmap_ram_data *data; + unsigned int rval[BLOCK_TEST_SIZE]; + int i; + + config = test_regmap_config; + config.num_reg_defaults = BLOCK_TEST_SIZE; + + map = gen_regmap(test, &config, &data); + KUNIT_ASSERT_FALSE(test, IS_ERR(map)); + if (IS_ERR(map)) + return; + + /* Ensure the data is read from the cache */ + for (i = 0; i < BLOCK_TEST_SIZE; i++) + data->read[param->from_reg + i] = false; + KUNIT_EXPECT_EQ(test, 0, regmap_bulk_read(map, param->from_reg, rval, + BLOCK_TEST_SIZE)); + KUNIT_EXPECT_MEMEQ(test, &data->vals[param->from_reg], rval, sizeof(rval)); + + /* Change all values in cache from defaults */ + for (i = 0; i < BLOCK_TEST_SIZE; i++) + KUNIT_EXPECT_EQ(test, 0, regmap_write(map, param->from_reg + i, rval[i] + 1)); + + /* Drop all registers */ + KUNIT_EXPECT_EQ(test, 0, regcache_drop_region(map, 0, config.max_register)); + + /* + * Sync cache without marking it dirty. All registers were dropped + * so the cache should not have any entries to write out. + */ + for (i = 0; i < BLOCK_TEST_SIZE; i++) + data->written[param->from_reg + i] = false; + + KUNIT_EXPECT_EQ(test, 0, regcache_sync(map)); + for (i = 0; i <= config.max_register; i++) + KUNIT_EXPECT_FALSE(test, data->written[i]); +} + +static void cache_present(struct kunit *test) +{ + const struct regmap_test_param *param = test->param_value; + struct regmap *map; + struct regmap_config config; + struct regmap_ram_data *data; + unsigned int val; + int i; + + config = test_regmap_config; + + map = gen_regmap(test, &config, &data); + KUNIT_ASSERT_FALSE(test, IS_ERR(map)); + if (IS_ERR(map)) + return; + + for (i = 0; i < BLOCK_TEST_SIZE; i++) + data->read[param->from_reg + i] = false; + + /* No defaults so no registers cached. */ + for (i = 0; i < BLOCK_TEST_SIZE; i++) + KUNIT_ASSERT_FALSE(test, regcache_reg_cached(map, param->from_reg + i)); + + /* We didn't trigger any reads */ + for (i = 0; i < BLOCK_TEST_SIZE; i++) + KUNIT_ASSERT_FALSE(test, data->read[param->from_reg + i]); + + /* Fill the cache */ + for (i = 0; i < BLOCK_TEST_SIZE; i++) + KUNIT_EXPECT_EQ(test, 0, regmap_read(map, param->from_reg + i, &val)); + + /* Now everything should be cached */ + for (i = 0; i < BLOCK_TEST_SIZE; i++) + KUNIT_ASSERT_TRUE(test, regcache_reg_cached(map, param->from_reg + i)); +} + +static void cache_write_zero(struct kunit *test) +{ + const struct regmap_test_param *param = test->param_value; + struct regmap *map; + struct regmap_config config; + struct regmap_ram_data *data; + unsigned int val; + int i; + + config = test_regmap_config; + + map = gen_regmap(test, &config, &data); + KUNIT_ASSERT_FALSE(test, IS_ERR(map)); + if (IS_ERR(map)) + return; + + for (i = 0; i < BLOCK_TEST_SIZE; i++) + data->read[param->from_reg + i] = false; + + /* No defaults so no registers cached. */ + for (i = 0; i < BLOCK_TEST_SIZE; i++) + KUNIT_ASSERT_FALSE(test, regcache_reg_cached(map, param->from_reg + i)); + + /* We didn't trigger any reads */ + for (i = 0; i < BLOCK_TEST_SIZE; i++) + KUNIT_ASSERT_FALSE(test, data->read[param->from_reg + i]); + + /* Write a zero value */ + KUNIT_EXPECT_EQ(test, 0, regmap_write(map, 1, 0)); + + /* Read that zero value back */ + KUNIT_EXPECT_EQ(test, 0, regmap_read(map, 1, &val)); + KUNIT_EXPECT_EQ(test, 0, val); + + /* From the cache? */ + KUNIT_ASSERT_TRUE(test, regcache_reg_cached(map, 1)); + + /* Try to throw it away */ + KUNIT_EXPECT_EQ(test, 0, regcache_drop_region(map, 1, 1)); + KUNIT_ASSERT_FALSE(test, regcache_reg_cached(map, 1)); +} + +/* Check that caching the window register works with sync */ +static void cache_range_window_reg(struct kunit *test) +{ + struct regmap *map; + struct regmap_config config; + struct regmap_ram_data *data; + unsigned int val; + int i; + + config = test_regmap_config; + config.volatile_reg = test_range_window_volatile; + config.ranges = &test_range; + config.num_ranges = 1; + config.max_register = test_range.range_max; + + map = gen_regmap(test, &config, &data); + KUNIT_ASSERT_FALSE(test, IS_ERR(map)); + if (IS_ERR(map)) + return; + + /* Write new values to the entire range */ + for (i = test_range.range_min; i <= test_range.range_max; i++) + KUNIT_ASSERT_EQ(test, 0, regmap_write(map, i, 0)); + + val = data->vals[test_range.selector_reg] & test_range.selector_mask; + KUNIT_ASSERT_EQ(test, val, 2); + + /* Write to the first register in the range to reset the page */ + KUNIT_ASSERT_EQ(test, 0, regmap_write(map, test_range.range_min, 0)); + val = data->vals[test_range.selector_reg] & test_range.selector_mask; + KUNIT_ASSERT_EQ(test, val, 0); + + /* Trigger a cache sync */ + regcache_mark_dirty(map); + KUNIT_ASSERT_EQ(test, 0, regcache_sync(map)); + + /* Write to the first register again, the page should be reset */ + KUNIT_ASSERT_EQ(test, 0, regmap_write(map, test_range.range_min, 0)); + val = data->vals[test_range.selector_reg] & test_range.selector_mask; + KUNIT_ASSERT_EQ(test, val, 0); + + /* Trigger another cache sync */ + regcache_mark_dirty(map); + KUNIT_ASSERT_EQ(test, 0, regcache_sync(map)); + + /* Write to the last register again, the page should be reset */ + KUNIT_ASSERT_EQ(test, 0, regmap_write(map, test_range.range_max, 0)); + val = data->vals[test_range.selector_reg] & test_range.selector_mask; + KUNIT_ASSERT_EQ(test, val, 2); +} + +static const struct regmap_test_param raw_types_list[] = { + { .cache = REGCACHE_NONE, .val_endian = REGMAP_ENDIAN_LITTLE }, + { .cache = REGCACHE_NONE, .val_endian = REGMAP_ENDIAN_BIG }, + { .cache = REGCACHE_FLAT, .val_endian = REGMAP_ENDIAN_LITTLE }, + { .cache = REGCACHE_FLAT, .val_endian = REGMAP_ENDIAN_BIG }, + { .cache = REGCACHE_FLAT_S, .val_endian = REGMAP_ENDIAN_LITTLE }, + { .cache = REGCACHE_FLAT_S, .val_endian = REGMAP_ENDIAN_BIG }, + { .cache = REGCACHE_RBTREE, .val_endian = REGMAP_ENDIAN_LITTLE }, + { .cache = REGCACHE_RBTREE, .val_endian = REGMAP_ENDIAN_BIG }, + { .cache = REGCACHE_MAPLE, .val_endian = REGMAP_ENDIAN_LITTLE }, + { .cache = REGCACHE_MAPLE, .val_endian = REGMAP_ENDIAN_BIG }, +}; + +KUNIT_ARRAY_PARAM(raw_test_types, raw_types_list, param_to_desc); + +static const struct regmap_test_param raw_cache_types_list[] = { + { .cache = REGCACHE_FLAT, .val_endian = REGMAP_ENDIAN_LITTLE }, + { .cache = REGCACHE_FLAT, .val_endian = REGMAP_ENDIAN_BIG }, + { .cache = REGCACHE_FLAT_S, .val_endian = REGMAP_ENDIAN_LITTLE }, + { .cache = REGCACHE_FLAT_S, .val_endian = REGMAP_ENDIAN_BIG }, + { .cache = REGCACHE_RBTREE, .val_endian = REGMAP_ENDIAN_LITTLE }, + { .cache = REGCACHE_RBTREE, .val_endian = REGMAP_ENDIAN_BIG }, + { .cache = REGCACHE_MAPLE, .val_endian = REGMAP_ENDIAN_LITTLE }, + { .cache = REGCACHE_MAPLE, .val_endian = REGMAP_ENDIAN_BIG }, +}; + +KUNIT_ARRAY_PARAM(raw_test_cache_types, raw_cache_types_list, param_to_desc); + +static const struct regmap_config raw_regmap_config = { + .max_register = BLOCK_TEST_SIZE, + + .reg_format_endian = REGMAP_ENDIAN_LITTLE, + .reg_bits = 16, + .val_bits = 16, +}; + +static struct regmap *gen_raw_regmap(struct kunit *test, + struct regmap_config *config, + struct regmap_ram_data **data) +{ + struct regmap_test_priv *priv = test->priv; + const struct regmap_test_param *param = test->param_value; + u16 *buf; + struct regmap *ret = ERR_PTR(-ENOMEM); + int i, error; + struct reg_default *defaults; + size_t size; + + config->cache_type = param->cache; + config->val_format_endian = param->val_endian; + config->disable_locking = config->cache_type == REGCACHE_RBTREE || + config->cache_type == REGCACHE_MAPLE; + + size = array_size(config->max_register + 1, BITS_TO_BYTES(config->reg_bits)); + buf = kmalloc(size, GFP_KERNEL); + if (!buf) + return ERR_PTR(-ENOMEM); + + get_random_bytes(buf, size); + + *data = kzalloc(sizeof(**data), GFP_KERNEL); + if (!(*data)) + goto out_free; + (*data)->vals = (void *)buf; + + config->num_reg_defaults = config->max_register + 1; + defaults = kunit_kcalloc(test, + config->num_reg_defaults, + sizeof(struct reg_default), + GFP_KERNEL); + if (!defaults) + goto out_free; + config->reg_defaults = defaults; + + for (i = 0; i < config->num_reg_defaults; i++) { + defaults[i].reg = i; + switch (param->val_endian) { + case REGMAP_ENDIAN_LITTLE: + defaults[i].def = le16_to_cpu(buf[i]); + break; + case REGMAP_ENDIAN_BIG: + defaults[i].def = be16_to_cpu(buf[i]); + break; + default: + ret = ERR_PTR(-EINVAL); + goto out_free; + } + } + + /* + * We use the defaults in the tests but they don't make sense + * to the core if there's no cache. + */ + if (config->cache_type == REGCACHE_NONE) + config->num_reg_defaults = 0; + + ret = regmap_init_raw_ram(priv->dev, config, *data); + if (IS_ERR(ret)) + goto out_free; + + /* This calls regmap_exit() on failure, which frees buf and *data */ + error = kunit_add_action_or_reset(test, regmap_exit_action, ret); + if (error) + ret = ERR_PTR(error); + + return ret; + +out_free: + kfree(buf); + kfree(*data); + + return ret; +} + +static void raw_read_defaults_single(struct kunit *test) +{ + struct regmap *map; + struct regmap_config config; + struct regmap_ram_data *data; + unsigned int rval; + int i; + + config = raw_regmap_config; + + map = gen_raw_regmap(test, &config, &data); + KUNIT_ASSERT_FALSE(test, IS_ERR(map)); + if (IS_ERR(map)) + return; + + /* Check that we can read the defaults via the API */ + for (i = 0; i < config.max_register + 1; i++) { + KUNIT_EXPECT_EQ(test, 0, regmap_read(map, i, &rval)); + KUNIT_EXPECT_EQ(test, config.reg_defaults[i].def, rval); + } +} + +static void raw_read_defaults(struct kunit *test) +{ + struct regmap *map; + struct regmap_config config; + struct regmap_ram_data *data; + u16 *rval; + u16 def; + size_t val_len; + int i; + + config = raw_regmap_config; + + map = gen_raw_regmap(test, &config, &data); + KUNIT_ASSERT_FALSE(test, IS_ERR(map)); + if (IS_ERR(map)) + return; + + val_len = array_size(sizeof(*rval), config.max_register + 1); + rval = kunit_kmalloc(test, val_len, GFP_KERNEL); + KUNIT_ASSERT_TRUE(test, rval != NULL); + if (!rval) + return; + + /* Check that we can read the defaults via the API */ + KUNIT_EXPECT_EQ(test, 0, regmap_raw_read(map, 0, rval, val_len)); + for (i = 0; i < config.max_register + 1; i++) { + def = config.reg_defaults[i].def; + if (config.val_format_endian == REGMAP_ENDIAN_BIG) { + KUNIT_EXPECT_EQ(test, def, be16_to_cpu((__force __be16)rval[i])); + } else { + KUNIT_EXPECT_EQ(test, def, le16_to_cpu((__force __le16)rval[i])); + } + } +} + +static void raw_write_read_single(struct kunit *test) +{ + struct regmap *map; + struct regmap_config config; + struct regmap_ram_data *data; + u16 val; + unsigned int rval; + + config = raw_regmap_config; + + map = gen_raw_regmap(test, &config, &data); + KUNIT_ASSERT_FALSE(test, IS_ERR(map)); + if (IS_ERR(map)) + return; + + get_random_bytes(&val, sizeof(val)); + + /* If we write a value to a register we can read it back */ + KUNIT_EXPECT_EQ(test, 0, regmap_write(map, 0, val)); + KUNIT_EXPECT_EQ(test, 0, regmap_read(map, 0, &rval)); + KUNIT_EXPECT_EQ(test, val, rval); +} + +static void raw_write(struct kunit *test) +{ + struct regmap *map; + struct regmap_config config; + struct regmap_ram_data *data; + u16 *hw_buf; + u16 val[2]; + unsigned int rval; + int i; + + config = raw_regmap_config; + + map = gen_raw_regmap(test, &config, &data); + KUNIT_ASSERT_FALSE(test, IS_ERR(map)); + if (IS_ERR(map)) + return; + + hw_buf = (u16 *)data->vals; + + get_random_bytes(&val, sizeof(val)); + + /* Do a raw write */ + KUNIT_EXPECT_EQ(test, 0, regmap_raw_write(map, 2, val, sizeof(val))); + + /* We should read back the new values, and defaults for the rest */ + for (i = 0; i < config.max_register + 1; i++) { + KUNIT_EXPECT_EQ(test, 0, regmap_read(map, i, &rval)); + + switch (i) { + case 2: + case 3: + if (config.val_format_endian == REGMAP_ENDIAN_BIG) { + KUNIT_EXPECT_EQ(test, rval, + be16_to_cpu((__force __be16)val[i % 2])); + } else { + KUNIT_EXPECT_EQ(test, rval, + le16_to_cpu((__force __le16)val[i % 2])); + } + break; + default: + KUNIT_EXPECT_EQ(test, config.reg_defaults[i].def, rval); + break; + } + } + + /* The values should appear in the "hardware" */ + KUNIT_EXPECT_MEMEQ(test, &hw_buf[2], val, sizeof(val)); +} + +static bool reg_zero(struct device *dev, unsigned int reg) +{ + return reg == 0; +} + +static bool ram_reg_zero(struct regmap_ram_data *data, unsigned int reg) +{ + return reg == 0; +} + +static void raw_noinc_write(struct kunit *test) +{ + struct regmap *map; + struct regmap_config config; + struct regmap_ram_data *data; + unsigned int val; + u16 val_test, val_last; + u16 val_array[BLOCK_TEST_SIZE]; + + config = raw_regmap_config; + config.volatile_reg = reg_zero; + config.writeable_noinc_reg = reg_zero; + config.readable_noinc_reg = reg_zero; + + map = gen_raw_regmap(test, &config, &data); + KUNIT_ASSERT_FALSE(test, IS_ERR(map)); + if (IS_ERR(map)) + return; + + data->noinc_reg = ram_reg_zero; + + get_random_bytes(&val_array, sizeof(val_array)); + + if (config.val_format_endian == REGMAP_ENDIAN_BIG) { + val_test = be16_to_cpu(val_array[1]) + 100; + val_last = be16_to_cpu(val_array[BLOCK_TEST_SIZE - 1]); + } else { + val_test = le16_to_cpu(val_array[1]) + 100; + val_last = le16_to_cpu(val_array[BLOCK_TEST_SIZE - 1]); + } + + /* Put some data into the register following the noinc register */ + KUNIT_EXPECT_EQ(test, 0, regmap_write(map, 1, val_test)); + + /* Write some data to the noinc register */ + KUNIT_EXPECT_EQ(test, 0, regmap_noinc_write(map, 0, val_array, + sizeof(val_array))); + + /* We should read back the last value written */ + KUNIT_EXPECT_EQ(test, 0, regmap_read(map, 0, &val)); + KUNIT_ASSERT_EQ(test, val_last, val); + + /* Make sure we didn't touch the register after the noinc register */ + KUNIT_EXPECT_EQ(test, 0, regmap_read(map, 1, &val)); + KUNIT_ASSERT_EQ(test, val_test, val); +} + +static void raw_sync(struct kunit *test) +{ + struct regmap *map; + struct regmap_config config; + struct regmap_ram_data *data; + u16 val[3]; + u16 *hw_buf; + unsigned int rval; + int i; + + config = raw_regmap_config; + + map = gen_raw_regmap(test, &config, &data); + KUNIT_ASSERT_FALSE(test, IS_ERR(map)); + if (IS_ERR(map)) + return; + + hw_buf = (u16 *)data->vals; + + get_changed_bytes(&hw_buf[2], &val[0], sizeof(val)); + + /* Do a regular write and a raw write in cache only mode */ + regcache_cache_only(map, true); + KUNIT_EXPECT_EQ(test, 0, regmap_raw_write(map, 2, val, + sizeof(u16) * 2)); + KUNIT_EXPECT_EQ(test, 0, regmap_write(map, 4, val[2])); + + /* We should read back the new values, and defaults for the rest */ + for (i = 0; i < config.max_register + 1; i++) { + KUNIT_EXPECT_EQ(test, 0, regmap_read(map, i, &rval)); + + switch (i) { + case 2: + case 3: + if (config.val_format_endian == REGMAP_ENDIAN_BIG) { + KUNIT_EXPECT_EQ(test, rval, + be16_to_cpu((__force __be16)val[i - 2])); + } else { + KUNIT_EXPECT_EQ(test, rval, + le16_to_cpu((__force __le16)val[i - 2])); + } + break; + case 4: + KUNIT_EXPECT_EQ(test, rval, val[i - 2]); + break; + default: + KUNIT_EXPECT_EQ(test, config.reg_defaults[i].def, rval); + break; + } + } + + /* + * The value written via _write() was translated by the core, + * translate the original copy for comparison purposes. + */ + if (config.val_format_endian == REGMAP_ENDIAN_BIG) + val[2] = cpu_to_be16(val[2]); + else + val[2] = cpu_to_le16(val[2]); + + /* The values should not appear in the "hardware" */ + KUNIT_EXPECT_MEMNEQ(test, &hw_buf[2], &val[0], sizeof(val)); + + for (i = 0; i < config.max_register + 1; i++) + data->written[i] = false; + + /* Do the sync */ + regcache_cache_only(map, false); + regcache_mark_dirty(map); + KUNIT_EXPECT_EQ(test, 0, regcache_sync(map)); + + /* The values should now appear in the "hardware" */ + KUNIT_EXPECT_MEMEQ(test, &hw_buf[2], &val[0], sizeof(val)); +} + +static void raw_ranges(struct kunit *test) +{ + struct regmap *map; + struct regmap_config config; + struct regmap_ram_data *data; + unsigned int val; + int i; + + config = raw_regmap_config; + config.volatile_reg = test_range_all_volatile; + config.ranges = &test_range; + config.num_ranges = 1; + config.max_register = test_range.range_max; + + map = gen_raw_regmap(test, &config, &data); + KUNIT_ASSERT_FALSE(test, IS_ERR(map)); + if (IS_ERR(map)) + return; + + /* Reset the page to a non-zero value to trigger a change */ + KUNIT_EXPECT_EQ(test, 0, regmap_write(map, test_range.selector_reg, + test_range.range_max)); + + /* Check we set the page and use the window for writes */ + data->written[test_range.selector_reg] = false; + data->written[test_range.window_start] = false; + KUNIT_EXPECT_EQ(test, 0, regmap_write(map, test_range.range_min, 0)); + KUNIT_EXPECT_TRUE(test, data->written[test_range.selector_reg]); + KUNIT_EXPECT_TRUE(test, data->written[test_range.window_start]); + + data->written[test_range.selector_reg] = false; + data->written[test_range.window_start] = false; + KUNIT_EXPECT_EQ(test, 0, regmap_write(map, + test_range.range_min + + test_range.window_len, + 0)); + KUNIT_EXPECT_TRUE(test, data->written[test_range.selector_reg]); + KUNIT_EXPECT_TRUE(test, data->written[test_range.window_start]); + + /* Same for reads */ + data->written[test_range.selector_reg] = false; + data->read[test_range.window_start] = false; + KUNIT_EXPECT_EQ(test, 0, regmap_read(map, test_range.range_min, &val)); + KUNIT_EXPECT_TRUE(test, data->written[test_range.selector_reg]); + KUNIT_EXPECT_TRUE(test, data->read[test_range.window_start]); + + data->written[test_range.selector_reg] = false; + data->read[test_range.window_start] = false; + KUNIT_EXPECT_EQ(test, 0, regmap_read(map, + test_range.range_min + + test_range.window_len, + &val)); + KUNIT_EXPECT_TRUE(test, data->written[test_range.selector_reg]); + KUNIT_EXPECT_TRUE(test, data->read[test_range.window_start]); + + /* No physical access triggered in the virtual range */ + for (i = test_range.range_min; i < test_range.range_max; i++) { + KUNIT_EXPECT_FALSE(test, data->read[i]); + KUNIT_EXPECT_FALSE(test, data->written[i]); + } +} + +static struct kunit_case regmap_test_cases[] = { + KUNIT_CASE_PARAM(basic_read_write, regcache_types_gen_params), + KUNIT_CASE_PARAM(read_bypassed, real_cache_types_gen_params), + KUNIT_CASE_PARAM(read_bypassed_volatile, real_cache_types_gen_params), + KUNIT_CASE_PARAM(bulk_write, regcache_types_gen_params), + KUNIT_CASE_PARAM(bulk_read, regcache_types_gen_params), + KUNIT_CASE_PARAM(multi_write, regcache_types_gen_params), + KUNIT_CASE_PARAM(multi_read, regcache_types_gen_params), + KUNIT_CASE_PARAM(write_readonly, regcache_types_gen_params), + KUNIT_CASE_PARAM(read_writeonly, regcache_types_gen_params), + KUNIT_CASE_PARAM(reg_defaults, regcache_types_gen_params), + KUNIT_CASE_PARAM(reg_defaults_read_dev, regcache_types_gen_params), + KUNIT_CASE_PARAM(register_patch, regcache_types_gen_params), + KUNIT_CASE_PARAM(stride, regcache_types_gen_params), + KUNIT_CASE_PARAM(basic_ranges, regcache_types_gen_params), + KUNIT_CASE_PARAM(stress_insert, regcache_types_gen_params), + KUNIT_CASE_PARAM(cache_bypass, real_cache_types_gen_params), + KUNIT_CASE_PARAM(cache_sync_marked_dirty, real_cache_types_gen_params), + KUNIT_CASE_PARAM(cache_sync_after_cache_only, real_cache_types_gen_params), + KUNIT_CASE_PARAM(cache_sync_defaults_marked_dirty, real_cache_types_gen_params), + KUNIT_CASE_PARAM(cache_sync_default_after_cache_only, real_cache_types_gen_params), + KUNIT_CASE_PARAM(cache_sync_readonly, real_cache_types_gen_params), + KUNIT_CASE_PARAM(cache_sync_patch, real_cache_types_gen_params), + KUNIT_CASE_PARAM(cache_drop, sparse_cache_types_gen_params), + KUNIT_CASE_PARAM(cache_drop_with_non_contiguous_ranges, sparse_cache_types_gen_params), + KUNIT_CASE_PARAM(cache_drop_all_and_sync_marked_dirty, sparse_cache_types_gen_params), + KUNIT_CASE_PARAM(cache_drop_all_and_sync_no_defaults, sparse_cache_types_gen_params), + KUNIT_CASE_PARAM(cache_drop_all_and_sync_has_defaults, sparse_cache_types_gen_params), + KUNIT_CASE_PARAM(cache_present, sparse_cache_types_gen_params), + KUNIT_CASE_PARAM(cache_write_zero, sparse_cache_types_gen_params), + KUNIT_CASE_PARAM(cache_range_window_reg, real_cache_types_only_gen_params), + + KUNIT_CASE_PARAM(raw_read_defaults_single, raw_test_types_gen_params), + KUNIT_CASE_PARAM(raw_read_defaults, raw_test_types_gen_params), + KUNIT_CASE_PARAM(raw_write_read_single, raw_test_types_gen_params), + KUNIT_CASE_PARAM(raw_write, raw_test_types_gen_params), + KUNIT_CASE_PARAM(raw_noinc_write, raw_test_types_gen_params), + KUNIT_CASE_PARAM(raw_sync, raw_test_cache_types_gen_params), + KUNIT_CASE_PARAM(raw_ranges, raw_test_cache_types_gen_params), + {} +}; + +static int regmap_test_init(struct kunit *test) +{ + struct regmap_test_priv *priv; + struct device *dev; + + priv = kunit_kzalloc(test, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + test->priv = priv; + + dev = kunit_device_register(test, "regmap_test"); + if (IS_ERR(dev)) + return PTR_ERR(dev); + + priv->dev = get_device(dev); + dev_set_drvdata(dev, test); + + return 0; +} + +static void regmap_test_exit(struct kunit *test) +{ + struct regmap_test_priv *priv = test->priv; + + /* Destroy the dummy struct device */ + if (priv && priv->dev) + put_device(priv->dev); +} + +static struct kunit_suite regmap_test_suite = { + .name = "regmap", + .init = regmap_test_init, + .exit = regmap_test_exit, + .test_cases = regmap_test_cases, +}; +kunit_test_suite(regmap_test_suite); + +MODULE_DESCRIPTION("Regmap KUnit tests"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/base/regmap/regmap-mdio.c b/drivers/base/regmap/regmap-mdio.c new file mode 100644 index 000000000000..9573bf3b52f4 --- /dev/null +++ b/drivers/base/regmap/regmap-mdio.c @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/errno.h> +#include <linux/mdio.h> +#include <linux/module.h> +#include <linux/regmap.h> + +#define REGVAL_MASK GENMASK(15, 0) +#define REGNUM_C22_MASK GENMASK(4, 0) +/* Clause-45 mask includes the device type (5 bit) and actual register number (16 bit) */ +#define REGNUM_C45_MASK GENMASK(20, 0) + +static int regmap_mdio_c22_read(void *context, unsigned int reg, unsigned int *val) +{ + struct mdio_device *mdio_dev = context; + int ret; + + if (unlikely(reg & ~REGNUM_C22_MASK)) + return -ENXIO; + + ret = mdiodev_read(mdio_dev, reg); + if (ret < 0) + return ret; + + *val = ret & REGVAL_MASK; + + return 0; +} + +static int regmap_mdio_c22_write(void *context, unsigned int reg, unsigned int val) +{ + struct mdio_device *mdio_dev = context; + + if (unlikely(reg & ~REGNUM_C22_MASK)) + return -ENXIO; + + return mdiodev_write(mdio_dev, reg, val); +} + +static const struct regmap_bus regmap_mdio_c22_bus = { + .reg_write = regmap_mdio_c22_write, + .reg_read = regmap_mdio_c22_read, +}; + +static int regmap_mdio_c45_read(void *context, unsigned int reg, unsigned int *val) +{ + struct mdio_device *mdio_dev = context; + unsigned int devad; + int ret; + + if (unlikely(reg & ~REGNUM_C45_MASK)) + return -ENXIO; + + devad = reg >> REGMAP_MDIO_C45_DEVAD_SHIFT; + reg = reg & REGMAP_MDIO_C45_REGNUM_MASK; + + ret = mdiodev_c45_read(mdio_dev, devad, reg); + if (ret < 0) + return ret; + + *val = ret & REGVAL_MASK; + + return 0; +} + +static int regmap_mdio_c45_write(void *context, unsigned int reg, unsigned int val) +{ + struct mdio_device *mdio_dev = context; + unsigned int devad; + + if (unlikely(reg & ~REGNUM_C45_MASK)) + return -ENXIO; + + devad = reg >> REGMAP_MDIO_C45_DEVAD_SHIFT; + reg = reg & REGMAP_MDIO_C45_REGNUM_MASK; + + return mdiodev_c45_write(mdio_dev, devad, reg, val); +} + +static const struct regmap_bus regmap_mdio_c45_bus = { + .reg_write = regmap_mdio_c45_write, + .reg_read = regmap_mdio_c45_read, +}; + +struct regmap *__regmap_init_mdio(struct mdio_device *mdio_dev, + const struct regmap_config *config, struct lock_class_key *lock_key, + const char *lock_name) +{ + const struct regmap_bus *bus; + + if (config->reg_bits == 5 && config->val_bits == 16) + bus = ®map_mdio_c22_bus; + else if (config->reg_bits == 21 && config->val_bits == 16) + bus = ®map_mdio_c45_bus; + else + return ERR_PTR(-EOPNOTSUPP); + + return __regmap_init(&mdio_dev->dev, bus, mdio_dev, config, lock_key, lock_name); +} +EXPORT_SYMBOL_GPL(__regmap_init_mdio); + +struct regmap *__devm_regmap_init_mdio(struct mdio_device *mdio_dev, + const struct regmap_config *config, struct lock_class_key *lock_key, + const char *lock_name) +{ + const struct regmap_bus *bus; + + if (config->reg_bits == 5 && config->val_bits == 16) + bus = ®map_mdio_c22_bus; + else if (config->reg_bits == 21 && config->val_bits == 16) + bus = ®map_mdio_c45_bus; + else + return ERR_PTR(-EOPNOTSUPP); + + return __devm_regmap_init(&mdio_dev->dev, bus, mdio_dev, config, lock_key, lock_name); +} +EXPORT_SYMBOL_GPL(__devm_regmap_init_mdio); + +MODULE_AUTHOR("Sander Vanheule <sander@svanheule.net>"); +MODULE_DESCRIPTION("regmap MDIO Module"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/base/regmap/regmap-mmio.c b/drivers/base/regmap/regmap-mmio.c index 98745dd77e8c..29e5f3175301 100644 --- a/drivers/base/regmap/regmap-mmio.c +++ b/drivers/base/regmap/regmap-mmio.c @@ -1,159 +1,407 @@ -/* - * Register map access API - MMIO support - * - * Copyright (c) 2012, NVIDIA CORPORATION. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms and conditions of the GNU General Public License, - * version 2, as published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - */ +// SPDX-License-Identifier: GPL-2.0 +// +// Register map access API - MMIO support +// +// Copyright (c) 2012, NVIDIA CORPORATION. All rights reserved. #include <linux/clk.h> #include <linux/err.h> -#include <linux/init.h> #include <linux/io.h> #include <linux/module.h> #include <linux/regmap.h> #include <linux/slab.h> +#include <linux/swab.h> + +#include "internal.h" struct regmap_mmio_context { void __iomem *regs; - unsigned val_bytes; + unsigned int val_bytes; + bool big_endian; + + bool attached_clk; struct clk *clk; + + void (*reg_write)(struct regmap_mmio_context *ctx, + unsigned int reg, unsigned int val); + unsigned int (*reg_read)(struct regmap_mmio_context *ctx, + unsigned int reg); }; -static int regmap_mmio_gather_write(void *context, - const void *reg, size_t reg_size, - const void *val, size_t val_size) +static int regmap_mmio_regbits_check(size_t reg_bits) +{ + switch (reg_bits) { + case 8: + case 16: + case 32: + return 0; + default: + return -EINVAL; + } +} + +static int regmap_mmio_get_min_stride(size_t val_bits) +{ + int min_stride; + + switch (val_bits) { + case 8: + /* The core treats 0 as 1 */ + min_stride = 0; + break; + case 16: + min_stride = 2; + break; + case 32: + min_stride = 4; + break; + default: + return -EINVAL; + } + + return min_stride; +} + +static void regmap_mmio_write8(struct regmap_mmio_context *ctx, + unsigned int reg, + unsigned int val) +{ + writeb(val, ctx->regs + reg); +} + +static void regmap_mmio_write8_relaxed(struct regmap_mmio_context *ctx, + unsigned int reg, + unsigned int val) +{ + writeb_relaxed(val, ctx->regs + reg); +} + +static void regmap_mmio_iowrite8(struct regmap_mmio_context *ctx, + unsigned int reg, unsigned int val) +{ + iowrite8(val, ctx->regs + reg); +} + +static void regmap_mmio_write16le(struct regmap_mmio_context *ctx, + unsigned int reg, + unsigned int val) +{ + writew(val, ctx->regs + reg); +} + +static void regmap_mmio_write16le_relaxed(struct regmap_mmio_context *ctx, + unsigned int reg, + unsigned int val) +{ + writew_relaxed(val, ctx->regs + reg); +} + +static void regmap_mmio_iowrite16le(struct regmap_mmio_context *ctx, + unsigned int reg, unsigned int val) +{ + iowrite16(val, ctx->regs + reg); +} + +static void regmap_mmio_write16be(struct regmap_mmio_context *ctx, + unsigned int reg, + unsigned int val) +{ + writew(swab16(val), ctx->regs + reg); +} + +static void regmap_mmio_iowrite16be(struct regmap_mmio_context *ctx, + unsigned int reg, unsigned int val) +{ + iowrite16be(val, ctx->regs + reg); +} + +static void regmap_mmio_write32le(struct regmap_mmio_context *ctx, + unsigned int reg, + unsigned int val) +{ + writel(val, ctx->regs + reg); +} + +static void regmap_mmio_write32le_relaxed(struct regmap_mmio_context *ctx, + unsigned int reg, + unsigned int val) +{ + writel_relaxed(val, ctx->regs + reg); +} + +static void regmap_mmio_iowrite32le(struct regmap_mmio_context *ctx, + unsigned int reg, unsigned int val) +{ + iowrite32(val, ctx->regs + reg); +} + +static void regmap_mmio_write32be(struct regmap_mmio_context *ctx, + unsigned int reg, + unsigned int val) +{ + writel(swab32(val), ctx->regs + reg); +} + +static void regmap_mmio_iowrite32be(struct regmap_mmio_context *ctx, + unsigned int reg, unsigned int val) +{ + iowrite32be(val, ctx->regs + reg); +} + +static int regmap_mmio_write(void *context, unsigned int reg, unsigned int val) { struct regmap_mmio_context *ctx = context; - u32 offset; int ret; - BUG_ON(reg_size != 4); - - if (ctx->clk) { + if (!IS_ERR(ctx->clk)) { ret = clk_enable(ctx->clk); if (ret < 0) return ret; } - offset = *(u32 *)reg; + ctx->reg_write(ctx, reg, val); + + if (!IS_ERR(ctx->clk)) + clk_disable(ctx->clk); + + return 0; +} + +static int regmap_mmio_noinc_write(void *context, unsigned int reg, + const void *val, size_t val_count) +{ + struct regmap_mmio_context *ctx = context; + int ret = 0; + int i; + + if (!IS_ERR(ctx->clk)) { + ret = clk_enable(ctx->clk); + if (ret < 0) + return ret; + } - while (val_size) { + /* + * There are no native, assembly-optimized write single register + * operations for big endian, so fall back to emulation if this + * is needed. (Single bytes are fine, they are not affected by + * endianness.) + */ + if (ctx->big_endian && (ctx->val_bytes > 1)) { switch (ctx->val_bytes) { - case 1: - writeb(*(u8 *)val, ctx->regs + offset); - break; case 2: - writew(*(u16 *)val, ctx->regs + offset); - break; + { + const u16 *valp = (const u16 *)val; + for (i = 0; i < val_count; i++) + writew(swab16(valp[i]), ctx->regs + reg); + goto out_clk; + } case 4: - writel(*(u32 *)val, ctx->regs + offset); - break; -#ifdef CONFIG_64BIT - case 8: - writeq(*(u64 *)val, ctx->regs + offset); - break; -#endif + { + const u32 *valp = (const u32 *)val; + for (i = 0; i < val_count; i++) + writel(swab32(valp[i]), ctx->regs + reg); + goto out_clk; + } default: - /* Should be caught by regmap_mmio_check_config */ - BUG(); + ret = -EINVAL; + goto out_clk; } - val_size -= ctx->val_bytes; - val += ctx->val_bytes; - offset += ctx->val_bytes; } - if (ctx->clk) + switch (ctx->val_bytes) { + case 1: + writesb(ctx->regs + reg, (const u8 *)val, val_count); + break; + case 2: + writesw(ctx->regs + reg, (const u16 *)val, val_count); + break; + case 4: + writesl(ctx->regs + reg, (const u32 *)val, val_count); + break; + default: + ret = -EINVAL; + break; + } + +out_clk: + if (!IS_ERR(ctx->clk)) clk_disable(ctx->clk); - return 0; + return ret; } -static int regmap_mmio_write(void *context, const void *data, size_t count) +static unsigned int regmap_mmio_read8(struct regmap_mmio_context *ctx, + unsigned int reg) { - BUG_ON(count < 4); + return readb(ctx->regs + reg); +} - return regmap_mmio_gather_write(context, data, 4, data + 4, count - 4); +static unsigned int regmap_mmio_read8_relaxed(struct regmap_mmio_context *ctx, + unsigned int reg) +{ + return readb_relaxed(ctx->regs + reg); } -static int regmap_mmio_read(void *context, - const void *reg, size_t reg_size, - void *val, size_t val_size) +static unsigned int regmap_mmio_ioread8(struct regmap_mmio_context *ctx, + unsigned int reg) +{ + return ioread8(ctx->regs + reg); +} + +static unsigned int regmap_mmio_read16le(struct regmap_mmio_context *ctx, + unsigned int reg) +{ + return readw(ctx->regs + reg); +} + +static unsigned int regmap_mmio_read16le_relaxed(struct regmap_mmio_context *ctx, + unsigned int reg) +{ + return readw_relaxed(ctx->regs + reg); +} + +static unsigned int regmap_mmio_ioread16le(struct regmap_mmio_context *ctx, + unsigned int reg) +{ + return ioread16(ctx->regs + reg); +} + +static unsigned int regmap_mmio_read16be(struct regmap_mmio_context *ctx, + unsigned int reg) +{ + return swab16(readw(ctx->regs + reg)); +} + +static unsigned int regmap_mmio_ioread16be(struct regmap_mmio_context *ctx, + unsigned int reg) +{ + return ioread16be(ctx->regs + reg); +} + +static unsigned int regmap_mmio_read32le(struct regmap_mmio_context *ctx, + unsigned int reg) +{ + return readl(ctx->regs + reg); +} + +static unsigned int regmap_mmio_read32le_relaxed(struct regmap_mmio_context *ctx, + unsigned int reg) +{ + return readl_relaxed(ctx->regs + reg); +} + +static unsigned int regmap_mmio_ioread32le(struct regmap_mmio_context *ctx, + unsigned int reg) +{ + return ioread32(ctx->regs + reg); +} + +static unsigned int regmap_mmio_read32be(struct regmap_mmio_context *ctx, + unsigned int reg) +{ + return swab32(readl(ctx->regs + reg)); +} + +static unsigned int regmap_mmio_ioread32be(struct regmap_mmio_context *ctx, + unsigned int reg) +{ + return ioread32be(ctx->regs + reg); +} + +static int regmap_mmio_read(void *context, unsigned int reg, unsigned int *val) { struct regmap_mmio_context *ctx = context; - u32 offset; int ret; - BUG_ON(reg_size != 4); + if (!IS_ERR(ctx->clk)) { + ret = clk_enable(ctx->clk); + if (ret < 0) + return ret; + } + + *val = ctx->reg_read(ctx, reg); + + if (!IS_ERR(ctx->clk)) + clk_disable(ctx->clk); + + return 0; +} + +static int regmap_mmio_noinc_read(void *context, unsigned int reg, + void *val, size_t val_count) +{ + struct regmap_mmio_context *ctx = context; + int ret = 0; - if (ctx->clk) { + if (!IS_ERR(ctx->clk)) { ret = clk_enable(ctx->clk); if (ret < 0) return ret; } - offset = *(u32 *)reg; + switch (ctx->val_bytes) { + case 1: + readsb(ctx->regs + reg, (u8 *)val, val_count); + break; + case 2: + readsw(ctx->regs + reg, (u16 *)val, val_count); + break; + case 4: + readsl(ctx->regs + reg, (u32 *)val, val_count); + break; + default: + ret = -EINVAL; + goto out_clk; + } - while (val_size) { + /* + * There are no native, assembly-optimized write single register + * operations for big endian, so fall back to emulation if this + * is needed. (Single bytes are fine, they are not affected by + * endianness.) + */ + if (ctx->big_endian && (ctx->val_bytes > 1)) { switch (ctx->val_bytes) { - case 1: - *(u8 *)val = readb(ctx->regs + offset); - break; case 2: - *(u16 *)val = readw(ctx->regs + offset); + swab16_array(val, val_count); break; case 4: - *(u32 *)val = readl(ctx->regs + offset); + swab32_array(val, val_count); break; -#ifdef CONFIG_64BIT - case 8: - *(u64 *)val = readq(ctx->regs + offset); - break; -#endif default: - /* Should be caught by regmap_mmio_check_config */ - BUG(); + ret = -EINVAL; + break; } - val_size -= ctx->val_bytes; - val += ctx->val_bytes; - offset += ctx->val_bytes; } - if (ctx->clk) +out_clk: + if (!IS_ERR(ctx->clk)) clk_disable(ctx->clk); - return 0; + return ret; } + static void regmap_mmio_free_context(void *context) { struct regmap_mmio_context *ctx = context; - if (ctx->clk) { + if (!IS_ERR(ctx->clk)) { clk_unprepare(ctx->clk); - clk_put(ctx->clk); + if (!ctx->attached_clk) + clk_put(ctx->clk); } kfree(context); } -static struct regmap_bus regmap_mmio = { +static const struct regmap_bus regmap_mmio = { .fast_io = true, - .write = regmap_mmio_write, - .gather_write = regmap_mmio_gather_write, - .read = regmap_mmio_read, + .reg_write = regmap_mmio_write, + .reg_read = regmap_mmio_read, + .reg_noinc_write = regmap_mmio_noinc_write, + .reg_noinc_read = regmap_mmio_noinc_read, .free_context = regmap_mmio_free_context, - .reg_format_endian_default = REGMAP_ENDIAN_NATIVE, - .val_format_endian_default = REGMAP_ENDIAN_NATIVE, + .val_format_endian_default = REGMAP_ENDIAN_LITTLE, }; static struct regmap_mmio_context *regmap_mmio_gen_context(struct device *dev, @@ -165,43 +413,22 @@ static struct regmap_mmio_context *regmap_mmio_gen_context(struct device *dev, int min_stride; int ret; - if (config->reg_bits != 32) - return ERR_PTR(-EINVAL); + ret = regmap_mmio_regbits_check(config->reg_bits); + if (ret) + return ERR_PTR(ret); if (config->pad_bits) return ERR_PTR(-EINVAL); - switch (config->val_bits) { - case 8: - /* The core treats 0 as 1 */ - min_stride = 0; - break; - case 16: - min_stride = 2; - break; - case 32: - min_stride = 4; - break; -#ifdef CONFIG_64BIT - case 64: - min_stride = 8; - break; -#endif - break; - default: - return ERR_PTR(-EINVAL); - } + min_stride = regmap_mmio_get_min_stride(config->val_bits); + if (min_stride < 0) + return ERR_PTR(min_stride); - if (config->reg_stride < min_stride) + if (config->reg_stride && config->reg_stride < min_stride) return ERR_PTR(-EINVAL); - switch (config->reg_format_endian) { - case REGMAP_ENDIAN_DEFAULT: - case REGMAP_ENDIAN_NATIVE: - break; - default: + if (config->use_relaxed_mmio && config->io_port) return ERR_PTR(-EINVAL); - } ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); if (!ctx) @@ -209,6 +436,98 @@ static struct regmap_mmio_context *regmap_mmio_gen_context(struct device *dev, ctx->regs = regs; ctx->val_bytes = config->val_bits / 8; + ctx->clk = ERR_PTR(-ENODEV); + + switch (regmap_get_val_endian(dev, ®map_mmio, config)) { + case REGMAP_ENDIAN_DEFAULT: + case REGMAP_ENDIAN_LITTLE: +#ifdef __LITTLE_ENDIAN + case REGMAP_ENDIAN_NATIVE: +#endif + switch (config->val_bits) { + case 8: + if (config->io_port) { + ctx->reg_read = regmap_mmio_ioread8; + ctx->reg_write = regmap_mmio_iowrite8; + } else if (config->use_relaxed_mmio) { + ctx->reg_read = regmap_mmio_read8_relaxed; + ctx->reg_write = regmap_mmio_write8_relaxed; + } else { + ctx->reg_read = regmap_mmio_read8; + ctx->reg_write = regmap_mmio_write8; + } + break; + case 16: + if (config->io_port) { + ctx->reg_read = regmap_mmio_ioread16le; + ctx->reg_write = regmap_mmio_iowrite16le; + } else if (config->use_relaxed_mmio) { + ctx->reg_read = regmap_mmio_read16le_relaxed; + ctx->reg_write = regmap_mmio_write16le_relaxed; + } else { + ctx->reg_read = regmap_mmio_read16le; + ctx->reg_write = regmap_mmio_write16le; + } + break; + case 32: + if (config->io_port) { + ctx->reg_read = regmap_mmio_ioread32le; + ctx->reg_write = regmap_mmio_iowrite32le; + } else if (config->use_relaxed_mmio) { + ctx->reg_read = regmap_mmio_read32le_relaxed; + ctx->reg_write = regmap_mmio_write32le_relaxed; + } else { + ctx->reg_read = regmap_mmio_read32le; + ctx->reg_write = regmap_mmio_write32le; + } + break; + default: + ret = -EINVAL; + goto err_free; + } + break; + case REGMAP_ENDIAN_BIG: +#ifdef __BIG_ENDIAN + case REGMAP_ENDIAN_NATIVE: +#endif + ctx->big_endian = true; + switch (config->val_bits) { + case 8: + if (config->io_port) { + ctx->reg_read = regmap_mmio_ioread8; + ctx->reg_write = regmap_mmio_iowrite8; + } else { + ctx->reg_read = regmap_mmio_read8; + ctx->reg_write = regmap_mmio_write8; + } + break; + case 16: + if (config->io_port) { + ctx->reg_read = regmap_mmio_ioread16be; + ctx->reg_write = regmap_mmio_iowrite16be; + } else { + ctx->reg_read = regmap_mmio_read16be; + ctx->reg_write = regmap_mmio_write16be; + } + break; + case 32: + if (config->io_port) { + ctx->reg_read = regmap_mmio_ioread32be; + ctx->reg_write = regmap_mmio_iowrite32be; + } else { + ctx->reg_read = regmap_mmio_read32be; + ctx->reg_write = regmap_mmio_write32be; + } + break; + default: + ret = -EINVAL; + goto err_free; + } + break; + default: + ret = -EINVAL; + goto err_free; + } if (clk_id == NULL) return ctx; @@ -233,20 +552,11 @@ err_free: return ERR_PTR(ret); } -/** - * regmap_init_mmio_clk(): Initialise register map with register clock - * - * @dev: Device that will be interacted with - * @clk_id: register clock consumer ID - * @regs: Pointer to memory-mapped IO region - * @config: Configuration for register map - * - * The return value will be an ERR_PTR() on error or a valid pointer to - * a struct regmap. - */ -struct regmap *regmap_init_mmio_clk(struct device *dev, const char *clk_id, - void __iomem *regs, - const struct regmap_config *config) +struct regmap *__regmap_init_mmio_clk(struct device *dev, const char *clk_id, + void __iomem *regs, + const struct regmap_config *config, + struct lock_class_key *lock_key, + const char *lock_name) { struct regmap_mmio_context *ctx; @@ -254,25 +564,17 @@ struct regmap *regmap_init_mmio_clk(struct device *dev, const char *clk_id, if (IS_ERR(ctx)) return ERR_CAST(ctx); - return regmap_init(dev, ®map_mmio, ctx, config); -} -EXPORT_SYMBOL_GPL(regmap_init_mmio_clk); - -/** - * devm_regmap_init_mmio_clk(): Initialise managed register map with clock - * - * @dev: Device that will be interacted with - * @clk_id: register clock consumer ID - * @regs: Pointer to memory-mapped IO region - * @config: Configuration for register map - * - * The return value will be an ERR_PTR() on error or a valid pointer - * to a struct regmap. The regmap will be automatically freed by the - * device management code. - */ -struct regmap *devm_regmap_init_mmio_clk(struct device *dev, const char *clk_id, - void __iomem *regs, - const struct regmap_config *config) + return __regmap_init(dev, ®map_mmio, ctx, config, + lock_key, lock_name); +} +EXPORT_SYMBOL_GPL(__regmap_init_mmio_clk); + +struct regmap *__devm_regmap_init_mmio_clk(struct device *dev, + const char *clk_id, + void __iomem *regs, + const struct regmap_config *config, + struct lock_class_key *lock_key, + const char *lock_name) { struct regmap_mmio_context *ctx; @@ -280,8 +582,32 @@ struct regmap *devm_regmap_init_mmio_clk(struct device *dev, const char *clk_id, if (IS_ERR(ctx)) return ERR_CAST(ctx); - return devm_regmap_init(dev, ®map_mmio, ctx, config); + return __devm_regmap_init(dev, ®map_mmio, ctx, config, + lock_key, lock_name); +} +EXPORT_SYMBOL_GPL(__devm_regmap_init_mmio_clk); + +int regmap_mmio_attach_clk(struct regmap *map, struct clk *clk) +{ + struct regmap_mmio_context *ctx = map->bus_context; + + ctx->clk = clk; + ctx->attached_clk = true; + + return clk_prepare(ctx->clk); +} +EXPORT_SYMBOL_GPL(regmap_mmio_attach_clk); + +void regmap_mmio_detach_clk(struct regmap *map) +{ + struct regmap_mmio_context *ctx = map->bus_context; + + clk_unprepare(ctx->clk); + + ctx->attached_clk = false; + ctx->clk = NULL; } -EXPORT_SYMBOL_GPL(devm_regmap_init_mmio_clk); +EXPORT_SYMBOL_GPL(regmap_mmio_detach_clk); +MODULE_DESCRIPTION("regmap MMIO Module"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/base/regmap/regmap-ram.c b/drivers/base/regmap/regmap-ram.c new file mode 100644 index 000000000000..4e5b4518ce4d --- /dev/null +++ b/drivers/base/regmap/regmap-ram.c @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: GPL-2.0 +// +// Register map access API - Memory region +// +// This is intended for testing only +// +// Copyright (c) 2023, Arm Ltd + +#include <linux/clk.h> +#include <linux/err.h> +#include <linux/io.h> +#include <linux/module.h> +#include <linux/regmap.h> +#include <linux/slab.h> +#include <linux/swab.h> + +#include "internal.h" + +static int regmap_ram_write(void *context, unsigned int reg, unsigned int val) +{ + struct regmap_ram_data *data = context; + + data->vals[reg] = val; + data->written[reg] = true; + + return 0; +} + +static int regmap_ram_read(void *context, unsigned int reg, unsigned int *val) +{ + struct regmap_ram_data *data = context; + + *val = data->vals[reg]; + data->read[reg] = true; + + return 0; +} + +static void regmap_ram_free_context(void *context) +{ + struct regmap_ram_data *data = context; + + kfree(data->vals); + kfree(data->read); + kfree(data->written); + kfree(data); +} + +static const struct regmap_bus regmap_ram = { + .fast_io = true, + .reg_write = regmap_ram_write, + .reg_read = regmap_ram_read, + .free_context = regmap_ram_free_context, +}; + +struct regmap *__regmap_init_ram(struct device *dev, + const struct regmap_config *config, + struct regmap_ram_data *data, + struct lock_class_key *lock_key, + const char *lock_name) +{ + struct regmap *map; + + if (!config->max_register) { + pr_crit("No max_register specified for RAM regmap\n"); + return ERR_PTR(-EINVAL); + } + + data->read = kcalloc(config->max_register + 1, sizeof(bool), + GFP_KERNEL); + if (!data->read) + return ERR_PTR(-ENOMEM); + + data->written = kcalloc(config->max_register + 1, sizeof(bool), + GFP_KERNEL); + if (!data->written) + return ERR_PTR(-ENOMEM); + + map = __regmap_init(dev, ®map_ram, data, config, + lock_key, lock_name); + + return map; +} +EXPORT_SYMBOL_GPL(__regmap_init_ram); + +MODULE_DESCRIPTION("Register map access API - Memory region"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/base/regmap/regmap-raw-ram.c b/drivers/base/regmap/regmap-raw-ram.c new file mode 100644 index 000000000000..76c98814fb8a --- /dev/null +++ b/drivers/base/regmap/regmap-raw-ram.c @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: GPL-2.0 +// +// Register map access API - Memory region with raw access +// +// This is intended for testing only +// +// Copyright (c) 2023, Arm Ltd + +#include <linux/clk.h> +#include <linux/err.h> +#include <linux/io.h> +#include <linux/module.h> +#include <linux/regmap.h> +#include <linux/slab.h> +#include <linux/swab.h> + +#include "internal.h" + +static unsigned int decode_reg(enum regmap_endian endian, const void *reg) +{ + const u16 *r = reg; + + if (endian == REGMAP_ENDIAN_BIG) + return be16_to_cpu(*r); + else + return le16_to_cpu(*r); +} + +static int regmap_raw_ram_gather_write(void *context, + const void *reg, size_t reg_len, + const void *val, size_t val_len) +{ + struct regmap_ram_data *data = context; + unsigned int r; + u16 *our_buf = (u16 *)data->vals; + int i; + + if (reg_len != 2) + return -EINVAL; + if (val_len % 2) + return -EINVAL; + + r = decode_reg(data->reg_endian, reg); + if (data->noinc_reg && data->noinc_reg(data, r)) { + memcpy(&our_buf[r], val + val_len - 2, 2); + data->written[r] = true; + } else { + memcpy(&our_buf[r], val, val_len); + + for (i = 0; i < val_len / 2; i++) + data->written[r + i] = true; + } + + return 0; +} + +static int regmap_raw_ram_write(void *context, const void *data, size_t count) +{ + return regmap_raw_ram_gather_write(context, data, 2, + data + 2, count - 2); +} + +static int regmap_raw_ram_read(void *context, + const void *reg, size_t reg_len, + void *val, size_t val_len) +{ + struct regmap_ram_data *data = context; + unsigned int r; + u16 *our_buf = (u16 *)data->vals; + int i; + + if (reg_len != 2) + return -EINVAL; + if (val_len % 2) + return -EINVAL; + + r = decode_reg(data->reg_endian, reg); + if (data->noinc_reg && data->noinc_reg(data, r)) { + for (i = 0; i < val_len; i += 2) + memcpy(val + i, &our_buf[r], 2); + data->read[r] = true; + } else { + memcpy(val, &our_buf[r], val_len); + + for (i = 0; i < val_len / 2; i++) + data->read[r + i] = true; + } + + return 0; +} + +static void regmap_raw_ram_free_context(void *context) +{ + struct regmap_ram_data *data = context; + + kfree(data->vals); + kfree(data->read); + kfree(data->written); + kfree(data); +} + +static const struct regmap_bus regmap_raw_ram = { + .fast_io = true, + .write = regmap_raw_ram_write, + .gather_write = regmap_raw_ram_gather_write, + .read = regmap_raw_ram_read, + .free_context = regmap_raw_ram_free_context, +}; + +struct regmap *__regmap_init_raw_ram(struct device *dev, + const struct regmap_config *config, + struct regmap_ram_data *data, + struct lock_class_key *lock_key, + const char *lock_name) +{ + struct regmap *map; + + if (config->reg_bits != 16) + return ERR_PTR(-EINVAL); + + if (!config->max_register) { + pr_crit("No max_register specified for RAM regmap\n"); + return ERR_PTR(-EINVAL); + } + + data->read = kcalloc(config->max_register + 1, sizeof(bool), + GFP_KERNEL); + if (!data->read) + return ERR_PTR(-ENOMEM); + + data->written = kcalloc(config->max_register + 1, sizeof(bool), + GFP_KERNEL); + if (!data->written) + return ERR_PTR(-ENOMEM); + + data->reg_endian = config->reg_format_endian; + + map = __regmap_init(dev, ®map_raw_ram, data, config, + lock_key, lock_name); + + return map; +} +EXPORT_SYMBOL_GPL(__regmap_init_raw_ram); + +MODULE_DESCRIPTION("Register map access API - Memory region with raw access"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/base/regmap/regmap-sccb.c b/drivers/base/regmap/regmap-sccb.c new file mode 100644 index 000000000000..12bbbb03e5f2 --- /dev/null +++ b/drivers/base/regmap/regmap-sccb.c @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: GPL-2.0 +// Register map access API - SCCB support + +#include <linux/i2c.h> +#include <linux/module.h> +#include <linux/regmap.h> + +#include "internal.h" + +/** + * sccb_is_available - Check if the adapter supports SCCB protocol + * @adap: I2C adapter + * + * Return true if the I2C adapter is capable of using SCCB helper functions, + * false otherwise. + */ +static bool sccb_is_available(struct i2c_adapter *adap) +{ + u32 needed_funcs = I2C_FUNC_SMBUS_BYTE | I2C_FUNC_SMBUS_WRITE_BYTE_DATA; + + /* + * If we ever want support for hardware doing SCCB natively, we will + * introduce a sccb_xfer() callback to struct i2c_algorithm and check + * for it here. + */ + + return (i2c_get_functionality(adap) & needed_funcs) == needed_funcs; +} + +/** + * regmap_sccb_read - Read data from SCCB slave device + * @context: Device that will be interacted with + * @reg: Register to be read from + * @val: Pointer to store read value + * + * This executes the 2-phase write transmission cycle that is followed by a + * 2-phase read transmission cycle, returning negative errno else zero on + * success. + */ +static int regmap_sccb_read(void *context, unsigned int reg, unsigned int *val) +{ + struct device *dev = context; + struct i2c_client *i2c = to_i2c_client(dev); + int ret; + union i2c_smbus_data data; + + i2c_lock_bus(i2c->adapter, I2C_LOCK_SEGMENT); + + ret = __i2c_smbus_xfer(i2c->adapter, i2c->addr, i2c->flags, + I2C_SMBUS_WRITE, reg, I2C_SMBUS_BYTE, NULL); + if (ret < 0) + goto out; + + ret = __i2c_smbus_xfer(i2c->adapter, i2c->addr, i2c->flags, + I2C_SMBUS_READ, 0, I2C_SMBUS_BYTE, &data); + if (ret < 0) + goto out; + + *val = data.byte; +out: + i2c_unlock_bus(i2c->adapter, I2C_LOCK_SEGMENT); + + return ret; +} + +/** + * regmap_sccb_write - Write data to SCCB slave device + * @context: Device that will be interacted with + * @reg: Register to write to + * @val: Value to be written + * + * This executes the SCCB 3-phase write transmission cycle, returning negative + * errno else zero on success. + */ +static int regmap_sccb_write(void *context, unsigned int reg, unsigned int val) +{ + struct device *dev = context; + struct i2c_client *i2c = to_i2c_client(dev); + + return i2c_smbus_write_byte_data(i2c, reg, val); +} + +static const struct regmap_bus regmap_sccb_bus = { + .reg_write = regmap_sccb_write, + .reg_read = regmap_sccb_read, +}; + +static const struct regmap_bus *regmap_get_sccb_bus(struct i2c_client *i2c, + const struct regmap_config *config) +{ + if (config->val_bits == 8 && config->reg_bits == 8 && + sccb_is_available(i2c->adapter)) + return ®map_sccb_bus; + + return ERR_PTR(-ENOTSUPP); +} + +struct regmap *__regmap_init_sccb(struct i2c_client *i2c, + const struct regmap_config *config, + struct lock_class_key *lock_key, + const char *lock_name) +{ + const struct regmap_bus *bus = regmap_get_sccb_bus(i2c, config); + + if (IS_ERR(bus)) + return ERR_CAST(bus); + + return __regmap_init(&i2c->dev, bus, &i2c->dev, config, + lock_key, lock_name); +} +EXPORT_SYMBOL_GPL(__regmap_init_sccb); + +struct regmap *__devm_regmap_init_sccb(struct i2c_client *i2c, + const struct regmap_config *config, + struct lock_class_key *lock_key, + const char *lock_name) +{ + const struct regmap_bus *bus = regmap_get_sccb_bus(i2c, config); + + if (IS_ERR(bus)) + return ERR_CAST(bus); + + return __devm_regmap_init(&i2c->dev, bus, &i2c->dev, config, + lock_key, lock_name); +} +EXPORT_SYMBOL_GPL(__devm_regmap_init_sccb); + +MODULE_DESCRIPTION("Register map access API - SCCB support"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/base/regmap/regmap-sdw-mbq.c b/drivers/base/regmap/regmap-sdw-mbq.c new file mode 100644 index 000000000000..6a61629f5f89 --- /dev/null +++ b/drivers/base/regmap/regmap-sdw-mbq.c @@ -0,0 +1,278 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2020 Intel Corporation. + +#include <linux/bits.h> +#include <linux/delay.h> +#include <linux/device.h> +#include <linux/errno.h> +#include <linux/iopoll.h> +#include <linux/module.h> +#include <linux/regmap.h> +#include <linux/soundwire/sdw.h> +#include <linux/soundwire/sdw_registers.h> +#include <sound/sdca_function.h> +#include "internal.h" + +struct regmap_mbq_context { + struct device *dev; + struct sdw_slave *sdw; + + bool (*readable_reg)(struct device *dev, unsigned int reg); + + struct regmap_sdw_mbq_cfg cfg; + + int val_size; +}; + +static int regmap_sdw_mbq_size(struct regmap_mbq_context *ctx, unsigned int reg) +{ + int size = ctx->val_size; + + if (ctx->cfg.mbq_size) { + size = ctx->cfg.mbq_size(ctx->dev, reg); + if (!size || size > ctx->val_size) + return -EINVAL; + } + + return size; +} + +static bool regmap_sdw_mbq_deferrable(struct regmap_mbq_context *ctx, unsigned int reg) +{ + if (ctx->cfg.deferrable) + return ctx->cfg.deferrable(ctx->dev, reg); + + return false; +} + +static int regmap_sdw_mbq_poll_busy(struct sdw_slave *slave, unsigned int reg, + struct regmap_mbq_context *ctx) +{ + struct device *dev = ctx->dev; + int val, ret = 0; + + dev_dbg(dev, "Deferring transaction for 0x%x\n", reg); + + reg = SDW_SDCA_CTL(SDW_SDCA_CTL_FUNC(reg), 0, + SDCA_CTL_ENTITY_0_FUNCTION_STATUS, 0); + + if (ctx->readable_reg(dev, reg)) { + ret = read_poll_timeout(sdw_read_no_pm, val, + val < 0 || !(val & SDCA_CTL_ENTITY_0_FUNCTION_BUSY), + ctx->cfg.timeout_us, ctx->cfg.retry_us, + false, slave, reg); + if (val < 0) + return val; + if (ret) + dev_err(dev, "Function busy timed out 0x%x: %d\n", reg, val); + } else { + fsleep(ctx->cfg.timeout_us); + } + + return ret; +} + +static int regmap_sdw_mbq_write_impl(struct sdw_slave *slave, + unsigned int reg, unsigned int val, + int mbq_size, bool deferrable) +{ + int shift = mbq_size * BITS_PER_BYTE; + int ret; + + while (--mbq_size > 0) { + shift -= BITS_PER_BYTE; + + ret = sdw_write_no_pm(slave, SDW_SDCA_MBQ_CTL(reg), + (val >> shift) & 0xff); + if (ret < 0) + return ret; + } + + ret = sdw_write_no_pm(slave, reg, val & 0xff); + if (deferrable && ret == -ENODATA) + return -EAGAIN; + + return ret; +} + +static int regmap_sdw_mbq_write(void *context, unsigned int reg, unsigned int val) +{ + struct regmap_mbq_context *ctx = context; + struct sdw_slave *slave = ctx->sdw; + bool deferrable = regmap_sdw_mbq_deferrable(ctx, reg); + int mbq_size = regmap_sdw_mbq_size(ctx, reg); + int ret; + + if (mbq_size < 0) + return mbq_size; + + /* + * Technically the spec does allow a device to set itself to busy for + * internal reasons, but since it doesn't provide any information on + * how to handle timeouts in that case, for now the code will only + * process a single wait/timeout on function busy and a single retry + * of the transaction. + */ + ret = regmap_sdw_mbq_write_impl(slave, reg, val, mbq_size, deferrable); + if (ret == -EAGAIN) { + ret = regmap_sdw_mbq_poll_busy(slave, reg, ctx); + if (ret) + return ret; + + ret = regmap_sdw_mbq_write_impl(slave, reg, val, mbq_size, false); + } + + return ret; +} + +static int regmap_sdw_mbq_read_impl(struct sdw_slave *slave, + unsigned int reg, unsigned int *val, + int mbq_size, bool deferrable) +{ + int shift = BITS_PER_BYTE; + int read; + + read = sdw_read_no_pm(slave, reg); + if (read < 0) { + if (deferrable && read == -ENODATA) + return -EAGAIN; + + return read; + } + + *val = read; + + while (--mbq_size > 0) { + read = sdw_read_no_pm(slave, SDW_SDCA_MBQ_CTL(reg)); + if (read < 0) + return read; + + *val |= read << shift; + shift += BITS_PER_BYTE; + } + + return 0; +} + +static int regmap_sdw_mbq_read(void *context, unsigned int reg, unsigned int *val) +{ + struct regmap_mbq_context *ctx = context; + struct sdw_slave *slave = ctx->sdw; + bool deferrable = regmap_sdw_mbq_deferrable(ctx, reg); + int mbq_size = regmap_sdw_mbq_size(ctx, reg); + int ret; + + if (mbq_size < 0) + return mbq_size; + + /* + * Technically the spec does allow a device to set itself to busy for + * internal reasons, but since it doesn't provide any information on + * how to handle timeouts in that case, for now the code will only + * process a single wait/timeout on function busy and a single retry + * of the transaction. + */ + ret = regmap_sdw_mbq_read_impl(slave, reg, val, mbq_size, deferrable); + if (ret == -EAGAIN) { + ret = regmap_sdw_mbq_poll_busy(slave, reg, ctx); + if (ret) + return ret; + + ret = regmap_sdw_mbq_read_impl(slave, reg, val, mbq_size, false); + } + + return ret; +} + +static const struct regmap_bus regmap_sdw_mbq = { + .reg_read = regmap_sdw_mbq_read, + .reg_write = regmap_sdw_mbq_write, + .reg_format_endian_default = REGMAP_ENDIAN_LITTLE, + .val_format_endian_default = REGMAP_ENDIAN_LITTLE, +}; + +static int regmap_sdw_mbq_config_check(const struct regmap_config *config) +{ + if (config->val_bits > (sizeof(unsigned int) * BITS_PER_BYTE)) + return -ENOTSUPP; + + /* Registers are 32 bits wide */ + if (config->reg_bits != 32) + return -ENOTSUPP; + + if (config->pad_bits != 0) + return -ENOTSUPP; + + return 0; +} + +static struct regmap_mbq_context * +regmap_sdw_mbq_gen_context(struct device *dev, + struct sdw_slave *sdw, + const struct regmap_config *config, + const struct regmap_sdw_mbq_cfg *mbq_config) +{ + struct regmap_mbq_context *ctx; + + ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return ERR_PTR(-ENOMEM); + + ctx->dev = dev; + ctx->sdw = sdw; + + if (mbq_config) + ctx->cfg = *mbq_config; + + ctx->val_size = config->val_bits / BITS_PER_BYTE; + ctx->readable_reg = config->readable_reg; + + return ctx; +} + +struct regmap *__regmap_init_sdw_mbq(struct device *dev, struct sdw_slave *sdw, + const struct regmap_config *config, + const struct regmap_sdw_mbq_cfg *mbq_config, + struct lock_class_key *lock_key, + const char *lock_name) +{ + struct regmap_mbq_context *ctx; + int ret; + + ret = regmap_sdw_mbq_config_check(config); + if (ret) + return ERR_PTR(ret); + + ctx = regmap_sdw_mbq_gen_context(dev, sdw, config, mbq_config); + if (IS_ERR(ctx)) + return ERR_CAST(ctx); + + return __regmap_init(dev, ®map_sdw_mbq, ctx, + config, lock_key, lock_name); +} +EXPORT_SYMBOL_GPL(__regmap_init_sdw_mbq); + +struct regmap *__devm_regmap_init_sdw_mbq(struct device *dev, struct sdw_slave *sdw, + const struct regmap_config *config, + const struct regmap_sdw_mbq_cfg *mbq_config, + struct lock_class_key *lock_key, + const char *lock_name) +{ + struct regmap_mbq_context *ctx; + int ret; + + ret = regmap_sdw_mbq_config_check(config); + if (ret) + return ERR_PTR(ret); + + ctx = regmap_sdw_mbq_gen_context(dev, sdw, config, mbq_config); + if (IS_ERR(ctx)) + return ERR_CAST(ctx); + + return __devm_regmap_init(dev, ®map_sdw_mbq, ctx, + config, lock_key, lock_name); +} +EXPORT_SYMBOL_GPL(__devm_regmap_init_sdw_mbq); + +MODULE_DESCRIPTION("regmap SoundWire MBQ Module"); +MODULE_LICENSE("GPL"); diff --git a/drivers/base/regmap/regmap-sdw.c b/drivers/base/regmap/regmap-sdw.c new file mode 100644 index 000000000000..ea631ac7c7ec --- /dev/null +++ b/drivers/base/regmap/regmap-sdw.c @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2015-17 Intel Corporation. + +#include <linux/device.h> +#include <linux/errno.h> +#include <linux/module.h> +#include <linux/regmap.h> +#include <linux/soundwire/sdw.h> +#include <linux/types.h> +#include "internal.h" + +static int regmap_sdw_write(void *context, const void *val_buf, size_t val_size) +{ + struct device *dev = context; + struct sdw_slave *slave = dev_to_sdw_dev(dev); + /* First word of buffer contains the destination address */ + u32 addr = le32_to_cpu(*(const __le32 *)val_buf); + const u8 *val = val_buf; + + return sdw_nwrite_no_pm(slave, addr, val_size - sizeof(addr), val + sizeof(addr)); +} + +static int regmap_sdw_gather_write(void *context, + const void *reg_buf, size_t reg_size, + const void *val_buf, size_t val_size) +{ + struct device *dev = context; + struct sdw_slave *slave = dev_to_sdw_dev(dev); + u32 addr = le32_to_cpu(*(const __le32 *)reg_buf); + + return sdw_nwrite_no_pm(slave, addr, val_size, val_buf); +} + +static int regmap_sdw_read(void *context, + const void *reg_buf, size_t reg_size, + void *val_buf, size_t val_size) +{ + struct device *dev = context; + struct sdw_slave *slave = dev_to_sdw_dev(dev); + u32 addr = le32_to_cpu(*(const __le32 *)reg_buf); + + return sdw_nread_no_pm(slave, addr, val_size, val_buf); +} + +static const struct regmap_bus regmap_sdw = { + .write = regmap_sdw_write, + .gather_write = regmap_sdw_gather_write, + .read = regmap_sdw_read, + .reg_format_endian_default = REGMAP_ENDIAN_LITTLE, + .val_format_endian_default = REGMAP_ENDIAN_LITTLE, +}; + +static int regmap_sdw_config_check(const struct regmap_config *config) +{ + /* Register addresses are 32 bits wide */ + if (config->reg_bits != 32) + return -ENOTSUPP; + + if (config->pad_bits != 0) + return -ENOTSUPP; + + /* Only bulk writes are supported not multi-register writes */ + if (config->can_multi_write) + return -ENOTSUPP; + + return 0; +} + +struct regmap *__regmap_init_sdw(struct sdw_slave *sdw, + const struct regmap_config *config, + struct lock_class_key *lock_key, + const char *lock_name) +{ + int ret; + + ret = regmap_sdw_config_check(config); + if (ret) + return ERR_PTR(ret); + + return __regmap_init(&sdw->dev, ®map_sdw, + &sdw->dev, config, lock_key, lock_name); +} +EXPORT_SYMBOL_GPL(__regmap_init_sdw); + +struct regmap *__devm_regmap_init_sdw(struct sdw_slave *sdw, + const struct regmap_config *config, + struct lock_class_key *lock_key, + const char *lock_name) +{ + int ret; + + ret = regmap_sdw_config_check(config); + if (ret) + return ERR_PTR(ret); + + return __devm_regmap_init(&sdw->dev, ®map_sdw, + &sdw->dev, config, lock_key, lock_name); +} +EXPORT_SYMBOL_GPL(__devm_regmap_init_sdw); + +MODULE_DESCRIPTION("regmap SoundWire Module"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/base/regmap/regmap-slimbus.c b/drivers/base/regmap/regmap-slimbus.c new file mode 100644 index 000000000000..e523fae73004 --- /dev/null +++ b/drivers/base/regmap/regmap-slimbus.c @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) 2017, Linaro Ltd. + +#include <linux/regmap.h> +#include <linux/slimbus.h> +#include <linux/module.h> + +#include "internal.h" + +static int regmap_slimbus_write(void *context, const void *data, size_t count) +{ + struct slim_device *sdev = context; + + return slim_write(sdev, *(u16 *)data, count - 2, (u8 *)data + 2); +} + +static int regmap_slimbus_read(void *context, const void *reg, size_t reg_size, + void *val, size_t val_size) +{ + struct slim_device *sdev = context; + + return slim_read(sdev, *(u16 *)reg, val_size, val); +} + +static const struct regmap_bus regmap_slimbus_bus = { + .write = regmap_slimbus_write, + .read = regmap_slimbus_read, + .reg_format_endian_default = REGMAP_ENDIAN_LITTLE, + .val_format_endian_default = REGMAP_ENDIAN_LITTLE, +}; + +static const struct regmap_bus *regmap_get_slimbus(struct slim_device *slim, + const struct regmap_config *config) +{ + if (config->val_bits == 8 && config->reg_bits == 16) + return ®map_slimbus_bus; + + return ERR_PTR(-ENOTSUPP); +} + +struct regmap *__regmap_init_slimbus(struct slim_device *slimbus, + const struct regmap_config *config, + struct lock_class_key *lock_key, + const char *lock_name) +{ + const struct regmap_bus *bus = regmap_get_slimbus(slimbus, config); + + if (IS_ERR(bus)) + return ERR_CAST(bus); + + return __regmap_init(&slimbus->dev, bus, slimbus, config, lock_key, lock_name); +} +EXPORT_SYMBOL_GPL(__regmap_init_slimbus); + +struct regmap *__devm_regmap_init_slimbus(struct slim_device *slimbus, + const struct regmap_config *config, + struct lock_class_key *lock_key, + const char *lock_name) +{ + const struct regmap_bus *bus = regmap_get_slimbus(slimbus, config); + + if (IS_ERR(bus)) + return ERR_CAST(bus); + + return __devm_regmap_init(&slimbus->dev, bus, slimbus, config, lock_key, lock_name); +} +EXPORT_SYMBOL_GPL(__devm_regmap_init_slimbus); + +MODULE_DESCRIPTION("Register map access API - SLIMbus support"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/base/regmap/regmap-spi-avmm.c b/drivers/base/regmap/regmap-spi-avmm.c new file mode 100644 index 000000000000..d86a06cadcdb --- /dev/null +++ b/drivers/base/regmap/regmap-spi-avmm.c @@ -0,0 +1,714 @@ +// SPDX-License-Identifier: GPL-2.0 +// +// Register map access API - SPI AVMM support +// +// Copyright (C) 2018-2020 Intel Corporation. All rights reserved. + +#include <linux/module.h> +#include <linux/regmap.h> +#include <linux/spi/spi.h> +#include <linux/swab.h> + +/* + * This driver implements the regmap operations for a generic SPI + * master to access the registers of the spi slave chip which has an + * Avalone bus in it. + * + * The "SPI slave to Avalon Master Bridge" (spi-avmm) IP should be integrated + * in the spi slave chip. The IP acts as a bridge to convert encoded streams of + * bytes from the host to the internal register read/write on Avalon bus. In + * order to issue register access requests to the slave chip, the host should + * send formatted bytes that conform to the transfer protocol. + * The transfer protocol contains 3 layers: transaction layer, packet layer + * and physical layer. + * + * Reference Documents could be found at: + * https://www.intel.com/content/www/us/en/programmable/documentation/sfo1400787952932.html + * + * Chapter "SPI Slave/JTAG to Avalon Master Bridge Cores" is a general + * introduction to the protocol. + * + * Chapter "Avalon Packets to Transactions Converter Core" describes + * the transaction layer. + * + * Chapter "Avalon-ST Bytes to Packets and Packets to Bytes Converter Cores" + * describes the packet layer. + * + * Chapter "Avalon-ST Serial Peripheral Interface Core" describes the + * physical layer. + * + * + * When host issues a regmap read/write, the driver will transform the request + * to byte stream layer by layer. It formats the register addr, value and + * length to the transaction layer request, then converts the request to packet + * layer bytes stream and then to physical layer bytes stream. Finally the + * driver sends the formatted byte stream over SPI bus to the slave chip. + * + * The spi-avmm IP on the slave chip decodes the byte stream and initiates + * register read/write on its internal Avalon bus, and then encodes the + * response to byte stream and sends back to host. + * + * The driver receives the byte stream, reverses the 3 layers transformation, + * and finally gets the response value (read out data for register read, + * successful written size for register write). + */ + +#define PKT_SOP 0x7a +#define PKT_EOP 0x7b +#define PKT_CHANNEL 0x7c +#define PKT_ESC 0x7d + +#define PHY_IDLE 0x4a +#define PHY_ESC 0x4d + +#define TRANS_CODE_WRITE 0x0 +#define TRANS_CODE_SEQ_WRITE 0x4 +#define TRANS_CODE_READ 0x10 +#define TRANS_CODE_SEQ_READ 0x14 +#define TRANS_CODE_NO_TRANS 0x7f + +#define SPI_AVMM_XFER_TIMEOUT (msecs_to_jiffies(200)) + +/* slave's register addr is 32 bits */ +#define SPI_AVMM_REG_SIZE 4UL +/* slave's register value is 32 bits */ +#define SPI_AVMM_VAL_SIZE 4UL + +/* + * max rx size could be larger. But considering the buffer consuming, + * it is proper that we limit 1KB xfer at max. + */ +#define MAX_READ_CNT 256UL +#define MAX_WRITE_CNT 1UL + +struct trans_req_header { + u8 code; + u8 rsvd; + __be16 size; + __be32 addr; +} __packed; + +struct trans_resp_header { + u8 r_code; + u8 rsvd; + __be16 size; +} __packed; + +#define TRANS_REQ_HD_SIZE (sizeof(struct trans_req_header)) +#define TRANS_RESP_HD_SIZE (sizeof(struct trans_resp_header)) + +/* + * In transaction layer, + * the write request format is: Transaction request header + data + * the read request format is: Transaction request header + * the write response format is: Transaction response header + * the read response format is: pure data, no Transaction response header + */ +#define TRANS_WR_TX_SIZE(n) (TRANS_REQ_HD_SIZE + SPI_AVMM_VAL_SIZE * (n)) +#define TRANS_RD_TX_SIZE TRANS_REQ_HD_SIZE +#define TRANS_TX_MAX TRANS_WR_TX_SIZE(MAX_WRITE_CNT) + +#define TRANS_RD_RX_SIZE(n) (SPI_AVMM_VAL_SIZE * (n)) +#define TRANS_WR_RX_SIZE TRANS_RESP_HD_SIZE +#define TRANS_RX_MAX TRANS_RD_RX_SIZE(MAX_READ_CNT) + +/* tx & rx share one transaction layer buffer */ +#define TRANS_BUF_SIZE ((TRANS_TX_MAX > TRANS_RX_MAX) ? \ + TRANS_TX_MAX : TRANS_RX_MAX) + +/* + * In tx phase, the host prepares all the phy layer bytes of a request in the + * phy buffer and sends them in a batch. + * + * The packet layer and physical layer defines several special chars for + * various purpose, when a transaction layer byte hits one of these special + * chars, it should be escaped. The escape rule is, "Escape char first, + * following the byte XOR'ed with 0x20". + * + * This macro defines the max possible length of the phy data. In the worst + * case, all transaction layer bytes need to be escaped (so the data length + * doubles), plus 4 special chars (SOP, CHANNEL, CHANNEL_NUM, EOP). Finally + * we should make sure the length is aligned to SPI BPW. + */ +#define PHY_TX_MAX ALIGN(2 * TRANS_TX_MAX + 4, 4) + +/* + * Unlike tx, phy rx is affected by possible PHY_IDLE bytes from slave, the max + * length of the rx bit stream is unpredictable. So the driver reads the words + * one by one, and parses each word immediately into transaction layer buffer. + * Only one word length of phy buffer is used for rx. + */ +#define PHY_BUF_SIZE PHY_TX_MAX + +/** + * struct spi_avmm_bridge - SPI slave to AVMM bus master bridge + * + * @spi: spi slave associated with this bridge. + * @word_len: bytes of word for spi transfer. + * @trans_len: length of valid data in trans_buf. + * @phy_len: length of valid data in phy_buf. + * @trans_buf: the bridge buffer for transaction layer data. + * @phy_buf: the bridge buffer for physical layer data. + * @swap_words: the word swapping cb for phy data. NULL if not needed. + * + * As a device's registers are implemented on the AVMM bus address space, it + * requires the driver to issue formatted requests to spi slave to AVMM bus + * master bridge to perform register access. + */ +struct spi_avmm_bridge { + struct spi_device *spi; + unsigned char word_len; + unsigned int trans_len; + unsigned int phy_len; + /* bridge buffer used in translation between protocol layers */ + char trans_buf[TRANS_BUF_SIZE]; + char phy_buf[PHY_BUF_SIZE]; + void (*swap_words)(void *buf, unsigned int len); +}; + +static void br_swap_words_32(void *buf, unsigned int len) +{ + swab32_array(buf, len / 4); +} + +/* + * Format transaction layer data in br->trans_buf according to the register + * access request, Store valid transaction layer data length in br->trans_len. + */ +static int br_trans_tx_prepare(struct spi_avmm_bridge *br, bool is_read, u32 reg, + u32 *wr_val, u32 count) +{ + struct trans_req_header *header; + unsigned int trans_len; + u8 code; + __le32 *data; + int i; + + if (is_read) { + if (count == 1) + code = TRANS_CODE_READ; + else + code = TRANS_CODE_SEQ_READ; + } else { + if (count == 1) + code = TRANS_CODE_WRITE; + else + code = TRANS_CODE_SEQ_WRITE; + } + + header = (struct trans_req_header *)br->trans_buf; + header->code = code; + header->rsvd = 0; + header->size = cpu_to_be16((u16)count * SPI_AVMM_VAL_SIZE); + header->addr = cpu_to_be32(reg); + + trans_len = TRANS_REQ_HD_SIZE; + + if (!is_read) { + trans_len += SPI_AVMM_VAL_SIZE * count; + if (trans_len > sizeof(br->trans_buf)) + return -ENOMEM; + + data = (__le32 *)(br->trans_buf + TRANS_REQ_HD_SIZE); + + for (i = 0; i < count; i++) + *data++ = cpu_to_le32(*wr_val++); + } + + /* Store valid trans data length for next layer */ + br->trans_len = trans_len; + + return 0; +} + +/* + * Convert transaction layer data (in br->trans_buf) to phy layer data, store + * them in br->phy_buf. Pad the phy_buf aligned with SPI's BPW. Store valid phy + * layer data length in br->phy_len. + * + * phy_buf len should be aligned with SPI's BPW. Spare bytes should be padded + * with PHY_IDLE, then the slave will just drop them. + * + * The driver will not simply pad 4a at the tail. The concern is that driver + * will not store MISO data during tx phase, if the driver pads 4a at the tail, + * it is possible that if the slave is fast enough to response at the padding + * time. As a result these rx bytes are lost. In the following case, 7a,7c,00 + * will lost. + * MOSI ...|7a|7c|00|10| |00|00|04|02| |4b|7d|5a|7b| |40|4a|4a|4a| |XX|XX|... + * MISO ...|4a|4a|4a|4a| |4a|4a|4a|4a| |4a|4a|4a|4a| |4a|7a|7c|00| |78|56|... + * + * So the driver moves EOP and bytes after EOP to the end of the aligned size, + * then fill the hole with PHY_IDLE. As following: + * before pad ...|7a|7c|00|10| |00|00|04|02| |4b|7d|5a|7b| |40| + * after pad ...|7a|7c|00|10| |00|00|04|02| |4b|7d|5a|4a| |4a|4a|7b|40| + * Then if the slave will not get the entire packet before the tx phase is + * over, it can't responsed to anything either. + */ +static int br_pkt_phy_tx_prepare(struct spi_avmm_bridge *br) +{ + char *tb, *tb_end, *pb, *pb_limit, *pb_eop = NULL; + unsigned int aligned_phy_len, move_size; + bool need_esc = false; + + tb = br->trans_buf; + tb_end = tb + br->trans_len; + pb = br->phy_buf; + pb_limit = pb + ARRAY_SIZE(br->phy_buf); + + *pb++ = PKT_SOP; + + /* + * The driver doesn't support multiple channels so the channel number + * is always 0. + */ + *pb++ = PKT_CHANNEL; + *pb++ = 0x0; + + for (; pb < pb_limit && tb < tb_end; pb++) { + if (need_esc) { + *pb = *tb++ ^ 0x20; + need_esc = false; + continue; + } + + /* EOP should be inserted before the last valid char */ + if (tb == tb_end - 1 && !pb_eop) { + *pb = PKT_EOP; + pb_eop = pb; + continue; + } + + /* + * insert an ESCAPE char if the data value equals any special + * char. + */ + switch (*tb) { + case PKT_SOP: + case PKT_EOP: + case PKT_CHANNEL: + case PKT_ESC: + *pb = PKT_ESC; + need_esc = true; + break; + case PHY_IDLE: + case PHY_ESC: + *pb = PHY_ESC; + need_esc = true; + break; + default: + *pb = *tb++; + break; + } + } + + /* The phy buffer is used out but transaction layer data remains */ + if (tb < tb_end) + return -ENOMEM; + + /* Store valid phy data length for spi transfer */ + br->phy_len = pb - br->phy_buf; + + if (br->word_len == 1) + return 0; + + /* Do phy buf padding if word_len > 1 byte. */ + aligned_phy_len = ALIGN(br->phy_len, br->word_len); + if (aligned_phy_len > sizeof(br->phy_buf)) + return -ENOMEM; + + if (aligned_phy_len == br->phy_len) + return 0; + + /* move EOP and bytes after EOP to the end of aligned size */ + move_size = pb - pb_eop; + memmove(&br->phy_buf[aligned_phy_len - move_size], pb_eop, move_size); + + /* fill the hole with PHY_IDLEs */ + memset(pb_eop, PHY_IDLE, aligned_phy_len - br->phy_len); + + /* update the phy data length */ + br->phy_len = aligned_phy_len; + + return 0; +} + +/* + * In tx phase, the slave only returns PHY_IDLE (0x4a). So the driver will + * ignore rx in tx phase. + */ +static int br_do_tx(struct spi_avmm_bridge *br) +{ + /* reorder words for spi transfer */ + if (br->swap_words) + br->swap_words(br->phy_buf, br->phy_len); + + /* send all data in phy_buf */ + return spi_write(br->spi, br->phy_buf, br->phy_len); +} + +/* + * This function read the rx byte stream from SPI word by word and convert + * them to transaction layer data in br->trans_buf. It also stores the length + * of rx transaction layer data in br->trans_len + * + * The slave may send an unknown number of PHY_IDLEs in rx phase, so we cannot + * prepare a fixed length buffer to receive all of the rx data in a batch. We + * have to read word by word and convert them to transaction layer data at + * once. + */ +static int br_do_rx_and_pkt_phy_parse(struct spi_avmm_bridge *br) +{ + bool eop_found = false, channel_found = false, esc_found = false; + bool valid_word = false, last_try = false; + struct device *dev = &br->spi->dev; + char *pb, *tb_limit, *tb = NULL; + unsigned long poll_timeout; + int ret, i; + + tb_limit = br->trans_buf + ARRAY_SIZE(br->trans_buf); + pb = br->phy_buf; + poll_timeout = jiffies + SPI_AVMM_XFER_TIMEOUT; + while (tb < tb_limit) { + ret = spi_read(br->spi, pb, br->word_len); + if (ret) + return ret; + + /* reorder the word back */ + if (br->swap_words) + br->swap_words(pb, br->word_len); + + valid_word = false; + for (i = 0; i < br->word_len; i++) { + /* drop everything before first SOP */ + if (!tb && pb[i] != PKT_SOP) + continue; + + /* drop PHY_IDLE */ + if (pb[i] == PHY_IDLE) + continue; + + valid_word = true; + + /* + * We don't support multiple channels, so error out if + * a non-zero channel number is found. + */ + if (channel_found) { + if (pb[i] != 0) { + dev_err(dev, "%s channel num != 0\n", + __func__); + return -EFAULT; + } + + channel_found = false; + continue; + } + + switch (pb[i]) { + case PKT_SOP: + /* + * reset the parsing if a second SOP appears. + */ + tb = br->trans_buf; + eop_found = false; + channel_found = false; + esc_found = false; + break; + case PKT_EOP: + /* + * No special char is expected after ESC char. + * No special char (except ESC & PHY_IDLE) is + * expected after EOP char. + * + * The special chars are all dropped. + */ + if (esc_found || eop_found) + return -EFAULT; + + eop_found = true; + break; + case PKT_CHANNEL: + if (esc_found || eop_found) + return -EFAULT; + + channel_found = true; + break; + case PKT_ESC: + case PHY_ESC: + if (esc_found) + return -EFAULT; + + esc_found = true; + break; + default: + /* Record the normal byte in trans_buf. */ + if (esc_found) { + *tb++ = pb[i] ^ 0x20; + esc_found = false; + } else { + *tb++ = pb[i]; + } + + /* + * We get the last normal byte after EOP, it is + * time we finish. Normally the function should + * return here. + */ + if (eop_found) { + br->trans_len = tb - br->trans_buf; + return 0; + } + } + } + + if (valid_word) { + /* update poll timeout when we get valid word */ + poll_timeout = jiffies + SPI_AVMM_XFER_TIMEOUT; + last_try = false; + } else { + /* + * We timeout when rx keeps invalid for some time. But + * it is possible we are scheduled out for long time + * after a spi_read. So when we are scheduled in, a SW + * timeout happens. But actually HW may have worked fine and + * has been ready long time ago. So we need to do an extra + * read, if we get a valid word then we could continue rx, + * otherwise real a HW issue happens. + */ + if (last_try) + return -ETIMEDOUT; + + if (time_after(jiffies, poll_timeout)) + last_try = true; + } + } + + /* + * We have used out all transfer layer buffer but cannot find the end + * of the byte stream. + */ + dev_err(dev, "%s transfer buffer is full but rx doesn't end\n", + __func__); + + return -EFAULT; +} + +/* + * For read transactions, the avmm bus will directly return register values + * without transaction response header. + */ +static int br_rd_trans_rx_parse(struct spi_avmm_bridge *br, + u32 *val, unsigned int expected_count) +{ + unsigned int i, trans_len = br->trans_len; + __le32 *data; + + if (expected_count * SPI_AVMM_VAL_SIZE != trans_len) + return -EFAULT; + + data = (__le32 *)br->trans_buf; + for (i = 0; i < expected_count; i++) + *val++ = le32_to_cpu(*data++); + + return 0; +} + +/* + * For write transactions, the slave will return a transaction response + * header. + */ +static int br_wr_trans_rx_parse(struct spi_avmm_bridge *br, + unsigned int expected_count) +{ + unsigned int trans_len = br->trans_len; + struct trans_resp_header *resp; + u8 code; + u16 val_len; + + if (trans_len != TRANS_RESP_HD_SIZE) + return -EFAULT; + + resp = (struct trans_resp_header *)br->trans_buf; + + code = resp->r_code ^ 0x80; + val_len = be16_to_cpu(resp->size); + if (!val_len || val_len != expected_count * SPI_AVMM_VAL_SIZE) + return -EFAULT; + + /* error out if the trans code doesn't align with the val size */ + if ((val_len == SPI_AVMM_VAL_SIZE && code != TRANS_CODE_WRITE) || + (val_len > SPI_AVMM_VAL_SIZE && code != TRANS_CODE_SEQ_WRITE)) + return -EFAULT; + + return 0; +} + +static int do_reg_access(void *context, bool is_read, unsigned int reg, + unsigned int *value, unsigned int count) +{ + struct spi_avmm_bridge *br = context; + int ret; + + /* invalidate bridge buffers first */ + br->trans_len = 0; + br->phy_len = 0; + + ret = br_trans_tx_prepare(br, is_read, reg, value, count); + if (ret) + return ret; + + ret = br_pkt_phy_tx_prepare(br); + if (ret) + return ret; + + ret = br_do_tx(br); + if (ret) + return ret; + + ret = br_do_rx_and_pkt_phy_parse(br); + if (ret) + return ret; + + if (is_read) + return br_rd_trans_rx_parse(br, value, count); + else + return br_wr_trans_rx_parse(br, count); +} + +static int regmap_spi_avmm_gather_write(void *context, + const void *reg_buf, size_t reg_len, + const void *val_buf, size_t val_len) +{ + if (reg_len != SPI_AVMM_REG_SIZE) + return -EINVAL; + + if (!IS_ALIGNED(val_len, SPI_AVMM_VAL_SIZE)) + return -EINVAL; + + return do_reg_access(context, false, *(u32 *)reg_buf, (u32 *)val_buf, + val_len / SPI_AVMM_VAL_SIZE); +} + +static int regmap_spi_avmm_write(void *context, const void *data, size_t bytes) +{ + if (bytes < SPI_AVMM_REG_SIZE + SPI_AVMM_VAL_SIZE) + return -EINVAL; + + return regmap_spi_avmm_gather_write(context, data, SPI_AVMM_REG_SIZE, + data + SPI_AVMM_REG_SIZE, + bytes - SPI_AVMM_REG_SIZE); +} + +static int regmap_spi_avmm_read(void *context, + const void *reg_buf, size_t reg_len, + void *val_buf, size_t val_len) +{ + if (reg_len != SPI_AVMM_REG_SIZE) + return -EINVAL; + + if (!IS_ALIGNED(val_len, SPI_AVMM_VAL_SIZE)) + return -EINVAL; + + return do_reg_access(context, true, *(u32 *)reg_buf, val_buf, + (val_len / SPI_AVMM_VAL_SIZE)); +} + +static struct spi_avmm_bridge * +spi_avmm_bridge_ctx_gen(struct spi_device *spi) +{ + struct spi_avmm_bridge *br; + + if (!spi) + return ERR_PTR(-ENODEV); + + /* Only support BPW == 8 or 32 now. Try 32 BPW first. */ + spi->mode = SPI_MODE_1; + spi->bits_per_word = 32; + if (spi_setup(spi)) { + spi->bits_per_word = 8; + if (spi_setup(spi)) + return ERR_PTR(-EINVAL); + } + + br = kzalloc(sizeof(*br), GFP_KERNEL); + if (!br) + return ERR_PTR(-ENOMEM); + + br->spi = spi; + br->word_len = spi->bits_per_word / 8; + if (br->word_len == 4) { + /* + * The protocol requires little endian byte order but MSB + * first. So driver needs to swap the byte order word by word + * if word length > 1. + */ + br->swap_words = br_swap_words_32; + } + + return br; +} + +static void spi_avmm_bridge_ctx_free(void *context) +{ + kfree(context); +} + +static const struct regmap_bus regmap_spi_avmm_bus = { + .write = regmap_spi_avmm_write, + .gather_write = regmap_spi_avmm_gather_write, + .read = regmap_spi_avmm_read, + .reg_format_endian_default = REGMAP_ENDIAN_NATIVE, + .val_format_endian_default = REGMAP_ENDIAN_NATIVE, + .max_raw_read = SPI_AVMM_VAL_SIZE * MAX_READ_CNT, + .max_raw_write = SPI_AVMM_VAL_SIZE * MAX_WRITE_CNT, + .free_context = spi_avmm_bridge_ctx_free, +}; + +struct regmap *__regmap_init_spi_avmm(struct spi_device *spi, + const struct regmap_config *config, + struct lock_class_key *lock_key, + const char *lock_name) +{ + struct spi_avmm_bridge *bridge; + struct regmap *map; + + bridge = spi_avmm_bridge_ctx_gen(spi); + if (IS_ERR(bridge)) + return ERR_CAST(bridge); + + map = __regmap_init(&spi->dev, ®map_spi_avmm_bus, + bridge, config, lock_key, lock_name); + if (IS_ERR(map)) { + spi_avmm_bridge_ctx_free(bridge); + return ERR_CAST(map); + } + + return map; +} +EXPORT_SYMBOL_GPL(__regmap_init_spi_avmm); + +struct regmap *__devm_regmap_init_spi_avmm(struct spi_device *spi, + const struct regmap_config *config, + struct lock_class_key *lock_key, + const char *lock_name) +{ + struct spi_avmm_bridge *bridge; + struct regmap *map; + + bridge = spi_avmm_bridge_ctx_gen(spi); + if (IS_ERR(bridge)) + return ERR_CAST(bridge); + + map = __devm_regmap_init(&spi->dev, ®map_spi_avmm_bus, + bridge, config, lock_key, lock_name); + if (IS_ERR(map)) { + spi_avmm_bridge_ctx_free(bridge); + return ERR_CAST(map); + } + + return map; +} +EXPORT_SYMBOL_GPL(__devm_regmap_init_spi_avmm); + +MODULE_DESCRIPTION("Register map access API - SPI AVMM support"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/base/regmap/regmap-spi.c b/drivers/base/regmap/regmap-spi.c index 4c506bd940f3..14b1d88997cb 100644 --- a/drivers/base/regmap/regmap-spi.c +++ b/drivers/base/regmap/regmap-spi.c @@ -1,18 +1,13 @@ -/* - * Register map access API - SPI support - * - * Copyright 2011 Wolfson Microelectronics plc - * - * Author: Mark Brown <broonie@opensource.wolfsonmicro.com> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - */ +// SPDX-License-Identifier: GPL-2.0 +// +// Register map access API - SPI support +// +// Copyright 2011 Wolfson Microelectronics plc +// +// Author: Mark Brown <broonie@opensource.wolfsonmicro.com> #include <linux/regmap.h> #include <linux/spi/spi.h> -#include <linux/init.h> #include <linux/module.h> #include "internal.h" @@ -73,7 +68,8 @@ static int regmap_spi_async_write(void *context, spi_message_init(&async->m); spi_message_add_tail(&async->t[0], &async->m); - spi_message_add_tail(&async->t[1], &async->m); + if (val) + spi_message_add_tail(&async->t[1], &async->m); async->m.complete = regmap_spi_complete; async->m.context = async; @@ -102,46 +98,71 @@ static int regmap_spi_read(void *context, return spi_write_then_read(spi, reg, reg_size, val, val_size); } -static struct regmap_bus regmap_spi = { +static const struct regmap_bus regmap_spi = { .write = regmap_spi_write, .gather_write = regmap_spi_gather_write, .async_write = regmap_spi_async_write, .async_alloc = regmap_spi_async_alloc, .read = regmap_spi_read, .read_flag_mask = 0x80, + .reg_format_endian_default = REGMAP_ENDIAN_BIG, + .val_format_endian_default = REGMAP_ENDIAN_BIG, }; -/** - * regmap_init_spi(): Initialise register map - * - * @spi: Device that will be interacted with - * @config: Configuration for register map - * - * The return value will be an ERR_PTR() on error or a valid pointer to - * a struct regmap. - */ -struct regmap *regmap_init_spi(struct spi_device *spi, - const struct regmap_config *config) +static const struct regmap_bus *regmap_get_spi_bus(struct spi_device *spi, + const struct regmap_config *config) { - return regmap_init(&spi->dev, ®map_spi, &spi->dev, config); + size_t max_size = spi_max_transfer_size(spi); + size_t max_msg_size, reg_reserve_size; + struct regmap_bus *bus; + + if (max_size != SIZE_MAX) { + bus = kmemdup(®map_spi, sizeof(*bus), GFP_KERNEL); + if (!bus) + return ERR_PTR(-ENOMEM); + + max_msg_size = spi_max_message_size(spi); + reg_reserve_size = (config->reg_bits + config->pad_bits) / BITS_PER_BYTE; + if (max_size + reg_reserve_size > max_msg_size) + max_size -= reg_reserve_size; + + bus->free_on_exit = true; + bus->max_raw_read = max_size; + bus->max_raw_write = max_size; + + return bus; + } + + return ®map_spi; } -EXPORT_SYMBOL_GPL(regmap_init_spi); - -/** - * devm_regmap_init_spi(): Initialise register map - * - * @spi: Device that will be interacted with - * @config: Configuration for register map - * - * The return value will be an ERR_PTR() on error or a valid pointer - * to a struct regmap. The map will be automatically freed by the - * device management code. - */ -struct regmap *devm_regmap_init_spi(struct spi_device *spi, - const struct regmap_config *config) + +struct regmap *__regmap_init_spi(struct spi_device *spi, + const struct regmap_config *config, + struct lock_class_key *lock_key, + const char *lock_name) { - return devm_regmap_init(&spi->dev, ®map_spi, &spi->dev, config); + const struct regmap_bus *bus = regmap_get_spi_bus(spi, config); + + if (IS_ERR(bus)) + return ERR_CAST(bus); + + return __regmap_init(&spi->dev, bus, &spi->dev, config, lock_key, lock_name); +} +EXPORT_SYMBOL_GPL(__regmap_init_spi); + +struct regmap *__devm_regmap_init_spi(struct spi_device *spi, + const struct regmap_config *config, + struct lock_class_key *lock_key, + const char *lock_name) +{ + const struct regmap_bus *bus = regmap_get_spi_bus(spi, config); + + if (IS_ERR(bus)) + return ERR_CAST(bus); + + return __devm_regmap_init(&spi->dev, bus, &spi->dev, config, lock_key, lock_name); } -EXPORT_SYMBOL_GPL(devm_regmap_init_spi); +EXPORT_SYMBOL_GPL(__devm_regmap_init_spi); +MODULE_DESCRIPTION("regmap SPI Module"); MODULE_LICENSE("GPL"); diff --git a/drivers/base/regmap/regmap-spmi.c b/drivers/base/regmap/regmap-spmi.c new file mode 100644 index 000000000000..347bfe9544ce --- /dev/null +++ b/drivers/base/regmap/regmap-spmi.c @@ -0,0 +1,226 @@ +// SPDX-License-Identifier: GPL-2.0 +// +// Register map access API - SPMI support +// +// Copyright (c) 2012-2013, The Linux Foundation. All rights reserved. +// +// Based on regmap-i2c.c: +// Copyright 2011 Wolfson Microelectronics plc +// Author: Mark Brown <broonie@opensource.wolfsonmicro.com> + +#include <linux/regmap.h> +#include <linux/spmi.h> +#include <linux/module.h> +#include <linux/init.h> + +static int regmap_spmi_base_read(void *context, + const void *reg, size_t reg_size, + void *val, size_t val_size) +{ + u8 addr = *(u8 *)reg; + int err = 0; + + BUG_ON(reg_size != 1); + + while (val_size-- && !err) + err = spmi_register_read(context, addr++, val++); + + return err; +} + +static int regmap_spmi_base_gather_write(void *context, + const void *reg, size_t reg_size, + const void *val, size_t val_size) +{ + const u8 *data = val; + u8 addr = *(u8 *)reg; + int err = 0; + + BUG_ON(reg_size != 1); + + /* + * SPMI defines a more bandwidth-efficient 'Register 0 Write' sequence, + * use it when possible. + */ + if (addr == 0 && val_size) { + err = spmi_register_zero_write(context, *data); + if (err) + goto err_out; + + data++; + addr++; + val_size--; + } + + while (val_size) { + err = spmi_register_write(context, addr, *data); + if (err) + goto err_out; + + data++; + addr++; + val_size--; + } + +err_out: + return err; +} + +static int regmap_spmi_base_write(void *context, const void *data, + size_t count) +{ + BUG_ON(count < 1); + return regmap_spmi_base_gather_write(context, data, 1, data + 1, + count - 1); +} + +static const struct regmap_bus regmap_spmi_base = { + .read = regmap_spmi_base_read, + .write = regmap_spmi_base_write, + .gather_write = regmap_spmi_base_gather_write, + .reg_format_endian_default = REGMAP_ENDIAN_NATIVE, + .val_format_endian_default = REGMAP_ENDIAN_NATIVE, +}; + +struct regmap *__regmap_init_spmi_base(struct spmi_device *sdev, + const struct regmap_config *config, + struct lock_class_key *lock_key, + const char *lock_name) +{ + return __regmap_init(&sdev->dev, ®map_spmi_base, sdev, config, + lock_key, lock_name); +} +EXPORT_SYMBOL_GPL(__regmap_init_spmi_base); + +struct regmap *__devm_regmap_init_spmi_base(struct spmi_device *sdev, + const struct regmap_config *config, + struct lock_class_key *lock_key, + const char *lock_name) +{ + return __devm_regmap_init(&sdev->dev, ®map_spmi_base, sdev, config, + lock_key, lock_name); +} +EXPORT_SYMBOL_GPL(__devm_regmap_init_spmi_base); + +static int regmap_spmi_ext_read(void *context, + const void *reg, size_t reg_size, + void *val, size_t val_size) +{ + int err = 0; + size_t len; + u16 addr; + + BUG_ON(reg_size != 2); + + addr = *(u16 *)reg; + + /* + * Split accesses into two to take advantage of the more + * bandwidth-efficient 'Extended Register Read' command when possible + */ + while (addr <= 0xFF && val_size) { + len = min_t(size_t, val_size, 16); + + err = spmi_ext_register_read(context, addr, val, len); + if (err) + goto err_out; + + addr += len; + val += len; + val_size -= len; + } + + while (val_size) { + len = min_t(size_t, val_size, 8); + + err = spmi_ext_register_readl(context, addr, val, len); + if (err) + goto err_out; + + addr += len; + val += len; + val_size -= len; + } + +err_out: + return err; +} + +static int regmap_spmi_ext_gather_write(void *context, + const void *reg, size_t reg_size, + const void *val, size_t val_size) +{ + int err = 0; + size_t len; + u16 addr; + + BUG_ON(reg_size != 2); + + addr = *(u16 *)reg; + + while (addr <= 0xFF && val_size) { + len = min_t(size_t, val_size, 16); + + err = spmi_ext_register_write(context, addr, val, len); + if (err) + goto err_out; + + addr += len; + val += len; + val_size -= len; + } + + while (val_size) { + len = min_t(size_t, val_size, 8); + + err = spmi_ext_register_writel(context, addr, val, len); + if (err) + goto err_out; + + addr += len; + val += len; + val_size -= len; + } + +err_out: + return err; +} + +static int regmap_spmi_ext_write(void *context, const void *data, + size_t count) +{ + BUG_ON(count < 2); + return regmap_spmi_ext_gather_write(context, data, 2, data + 2, + count - 2); +} + +static const struct regmap_bus regmap_spmi_ext = { + .read = regmap_spmi_ext_read, + .write = regmap_spmi_ext_write, + .gather_write = regmap_spmi_ext_gather_write, + .reg_format_endian_default = REGMAP_ENDIAN_NATIVE, + .val_format_endian_default = REGMAP_ENDIAN_NATIVE, +}; + +struct regmap *__regmap_init_spmi_ext(struct spmi_device *sdev, + const struct regmap_config *config, + struct lock_class_key *lock_key, + const char *lock_name) +{ + return __regmap_init(&sdev->dev, ®map_spmi_ext, sdev, config, + lock_key, lock_name); +} +EXPORT_SYMBOL_GPL(__regmap_init_spmi_ext); + +struct regmap *__devm_regmap_init_spmi_ext(struct spmi_device *sdev, + const struct regmap_config *config, + struct lock_class_key *lock_key, + const char *lock_name) +{ + return __devm_regmap_init(&sdev->dev, ®map_spmi_ext, sdev, config, + lock_key, lock_name); +} +EXPORT_SYMBOL_GPL(__devm_regmap_init_spmi_ext); + +MODULE_DESCRIPTION("Register map access API - SPMI support"); +MODULE_LICENSE("GPL"); diff --git a/drivers/base/regmap/regmap-w1.c b/drivers/base/regmap/regmap-w1.c new file mode 100644 index 000000000000..29fd24f9c7ed --- /dev/null +++ b/drivers/base/regmap/regmap-w1.c @@ -0,0 +1,238 @@ +// SPDX-License-Identifier: GPL-2.0 +// +// Register map access API - W1 (1-Wire) support +// +// Copyright (c) 2017 Radioavionica Corporation +// Author: Alex A. Mihaylov <minimumlaw@rambler.ru> + +#include <linux/regmap.h> +#include <linux/module.h> +#include <linux/w1.h> + +#include "internal.h" + +#define W1_CMD_READ_DATA 0x69 +#define W1_CMD_WRITE_DATA 0x6C + +/* + * 1-Wire slaves registers with addess 8 bit and data 8 bit + */ + +static int w1_reg_a8_v8_read(void *context, unsigned int reg, unsigned int *val) +{ + struct device *dev = context; + struct w1_slave *sl = container_of(dev, struct w1_slave, dev); + int ret = 0; + + if (reg > 255) + return -EINVAL; + + mutex_lock(&sl->master->bus_mutex); + if (!w1_reset_select_slave(sl)) { + w1_write_8(sl->master, W1_CMD_READ_DATA); + w1_write_8(sl->master, reg); + *val = w1_read_8(sl->master); + } else { + ret = -ENODEV; + } + mutex_unlock(&sl->master->bus_mutex); + + return ret; +} + +static int w1_reg_a8_v8_write(void *context, unsigned int reg, unsigned int val) +{ + struct device *dev = context; + struct w1_slave *sl = container_of(dev, struct w1_slave, dev); + int ret = 0; + + if (reg > 255) + return -EINVAL; + + mutex_lock(&sl->master->bus_mutex); + if (!w1_reset_select_slave(sl)) { + w1_write_8(sl->master, W1_CMD_WRITE_DATA); + w1_write_8(sl->master, reg); + w1_write_8(sl->master, val); + } else { + ret = -ENODEV; + } + mutex_unlock(&sl->master->bus_mutex); + + return ret; +} + +/* + * 1-Wire slaves registers with addess 8 bit and data 16 bit + */ + +static int w1_reg_a8_v16_read(void *context, unsigned int reg, + unsigned int *val) +{ + struct device *dev = context; + struct w1_slave *sl = container_of(dev, struct w1_slave, dev); + int ret = 0; + + if (reg > 255) + return -EINVAL; + + mutex_lock(&sl->master->bus_mutex); + if (!w1_reset_select_slave(sl)) { + w1_write_8(sl->master, W1_CMD_READ_DATA); + w1_write_8(sl->master, reg); + *val = w1_read_8(sl->master); + *val |= w1_read_8(sl->master)<<8; + } else { + ret = -ENODEV; + } + mutex_unlock(&sl->master->bus_mutex); + + return ret; +} + +static int w1_reg_a8_v16_write(void *context, unsigned int reg, + unsigned int val) +{ + struct device *dev = context; + struct w1_slave *sl = container_of(dev, struct w1_slave, dev); + int ret = 0; + + if (reg > 255) + return -EINVAL; + + mutex_lock(&sl->master->bus_mutex); + if (!w1_reset_select_slave(sl)) { + w1_write_8(sl->master, W1_CMD_WRITE_DATA); + w1_write_8(sl->master, reg); + w1_write_8(sl->master, val & 0x00FF); + w1_write_8(sl->master, val>>8 & 0x00FF); + } else { + ret = -ENODEV; + } + mutex_unlock(&sl->master->bus_mutex); + + return ret; +} + +/* + * 1-Wire slaves registers with addess 16 bit and data 16 bit + */ + +static int w1_reg_a16_v16_read(void *context, unsigned int reg, + unsigned int *val) +{ + struct device *dev = context; + struct w1_slave *sl = container_of(dev, struct w1_slave, dev); + int ret = 0; + + if (reg > 65535) + return -EINVAL; + + mutex_lock(&sl->master->bus_mutex); + if (!w1_reset_select_slave(sl)) { + w1_write_8(sl->master, W1_CMD_READ_DATA); + w1_write_8(sl->master, reg & 0x00FF); + w1_write_8(sl->master, reg>>8 & 0x00FF); + *val = w1_read_8(sl->master); + *val |= w1_read_8(sl->master)<<8; + } else { + ret = -ENODEV; + } + mutex_unlock(&sl->master->bus_mutex); + + return ret; +} + +static int w1_reg_a16_v16_write(void *context, unsigned int reg, + unsigned int val) +{ + struct device *dev = context; + struct w1_slave *sl = container_of(dev, struct w1_slave, dev); + int ret = 0; + + if (reg > 65535) + return -EINVAL; + + mutex_lock(&sl->master->bus_mutex); + if (!w1_reset_select_slave(sl)) { + w1_write_8(sl->master, W1_CMD_WRITE_DATA); + w1_write_8(sl->master, reg & 0x00FF); + w1_write_8(sl->master, reg>>8 & 0x00FF); + w1_write_8(sl->master, val & 0x00FF); + w1_write_8(sl->master, val>>8 & 0x00FF); + } else { + ret = -ENODEV; + } + mutex_unlock(&sl->master->bus_mutex); + + return ret; +} + +/* + * Various types of supported bus addressing + */ + +static const struct regmap_bus regmap_w1_bus_a8_v8 = { + .reg_read = w1_reg_a8_v8_read, + .reg_write = w1_reg_a8_v8_write, +}; + +static const struct regmap_bus regmap_w1_bus_a8_v16 = { + .reg_read = w1_reg_a8_v16_read, + .reg_write = w1_reg_a8_v16_write, +}; + +static const struct regmap_bus regmap_w1_bus_a16_v16 = { + .reg_read = w1_reg_a16_v16_read, + .reg_write = w1_reg_a16_v16_write, +}; + +static const struct regmap_bus *regmap_get_w1_bus(struct device *w1_dev, + const struct regmap_config *config) +{ + if (config->reg_bits == 8 && config->val_bits == 8) + return ®map_w1_bus_a8_v8; + + if (config->reg_bits == 8 && config->val_bits == 16) + return ®map_w1_bus_a8_v16; + + if (config->reg_bits == 16 && config->val_bits == 16) + return ®map_w1_bus_a16_v16; + + return ERR_PTR(-ENOTSUPP); +} + +struct regmap *__regmap_init_w1(struct device *w1_dev, + const struct regmap_config *config, + struct lock_class_key *lock_key, + const char *lock_name) +{ + + const struct regmap_bus *bus = regmap_get_w1_bus(w1_dev, config); + + if (IS_ERR(bus)) + return ERR_CAST(bus); + + return __regmap_init(w1_dev, bus, w1_dev, config, + lock_key, lock_name); +} +EXPORT_SYMBOL_GPL(__regmap_init_w1); + +struct regmap *__devm_regmap_init_w1(struct device *w1_dev, + const struct regmap_config *config, + struct lock_class_key *lock_key, + const char *lock_name) +{ + + const struct regmap_bus *bus = regmap_get_w1_bus(w1_dev, config); + + if (IS_ERR(bus)) + return ERR_CAST(bus); + + return __devm_regmap_init(w1_dev, bus, w1_dev, config, + lock_key, lock_name); +} +EXPORT_SYMBOL_GPL(__devm_regmap_init_w1); + +MODULE_DESCRIPTION("Register map access API - W1 (1-Wire) support"); +MODULE_LICENSE("GPL"); diff --git a/drivers/base/regmap/regmap.c b/drivers/base/regmap/regmap.c index 95920583e31e..ce9be3989a21 100644 --- a/drivers/base/regmap/regmap.c +++ b/drivers/base/regmap/regmap.c @@ -1,25 +1,26 @@ -/* - * Register map access API - * - * Copyright 2011 Wolfson Microelectronics plc - * - * Author: Mark Brown <broonie@opensource.wolfsonmicro.com> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - */ +// SPDX-License-Identifier: GPL-2.0 +// +// Register map access API +// +// Copyright 2011 Wolfson Microelectronics plc +// +// Author: Mark Brown <broonie@opensource.wolfsonmicro.com> #include <linux/device.h> #include <linux/slab.h> #include <linux/export.h> #include <linux/mutex.h> #include <linux/err.h> +#include <linux/property.h> #include <linux/rbtree.h> #include <linux/sched.h> +#include <linux/delay.h> +#include <linux/log2.h> +#include <linux/hwspinlock.h> +#include <linux/unaligned.h> #define CREATE_TRACE_POINTS -#include <trace/events/regmap.h> +#include "trace.h" #include "internal.h" @@ -31,26 +32,31 @@ */ #undef LOG_DEVICE +#ifdef LOG_DEVICE +static inline bool regmap_should_log(struct regmap *map) +{ + return (map->dev && strcmp(dev_name(map->dev), LOG_DEVICE) == 0); +} +#else +static inline bool regmap_should_log(struct regmap *map) { return false; } +#endif + + static int _regmap_update_bits(struct regmap *map, unsigned int reg, unsigned int mask, unsigned int val, - bool *change); + bool *change, bool force_write); +static int _regmap_bus_reg_read(void *context, unsigned int reg, + unsigned int *val); static int _regmap_bus_read(void *context, unsigned int reg, unsigned int *val); static int _regmap_bus_formatted_write(void *context, unsigned int reg, unsigned int val); +static int _regmap_bus_reg_write(void *context, unsigned int reg, + unsigned int val); static int _regmap_bus_raw_write(void *context, unsigned int reg, unsigned int val); -static void async_cleanup(struct work_struct *work) -{ - struct regmap_async *async = container_of(work, struct regmap_async, - cleanup); - - kfree(async->work_buf); - kfree(async); -} - bool regmap_reg_in_ranges(unsigned int reg, const struct regmap_range *ranges, unsigned int nranges) @@ -83,7 +89,7 @@ EXPORT_SYMBOL_GPL(regmap_check_range_table); bool regmap_writeable(struct regmap *map, unsigned int reg) { - if (map->max_register && reg > map->max_register) + if (map->max_register_is_set && reg > map->max_register) return false; if (map->writeable_reg) @@ -95,9 +101,35 @@ bool regmap_writeable(struct regmap *map, unsigned int reg) return true; } +bool regmap_cached(struct regmap *map, unsigned int reg) +{ + int ret; + unsigned int val; + + if (map->cache_type == REGCACHE_NONE) + return false; + + if (!map->cache_ops) + return false; + + if (map->max_register_is_set && reg > map->max_register) + return false; + + map->lock(map->lock_arg); + ret = regcache_read(map, reg, &val); + map->unlock(map->lock_arg); + if (ret) + return false; + + return true; +} + bool regmap_readable(struct regmap *map, unsigned int reg) { - if (map->max_register && reg > map->max_register) + if (!map->reg_read) + return false; + + if (map->max_register_is_set && reg > map->max_register) return false; if (map->format.format_write) @@ -114,7 +146,7 @@ bool regmap_readable(struct regmap *map, unsigned int reg) bool regmap_volatile(struct regmap *map, unsigned int reg) { - if (!regmap_readable(map, reg)) + if (!map->format.format_write && !regmap_readable(map, reg)) return false; if (map->volatile_reg) @@ -143,18 +175,52 @@ bool regmap_precious(struct regmap *map, unsigned int reg) return false; } +bool regmap_writeable_noinc(struct regmap *map, unsigned int reg) +{ + if (map->writeable_noinc_reg) + return map->writeable_noinc_reg(map->dev, reg); + + if (map->wr_noinc_table) + return regmap_check_range_table(map, reg, map->wr_noinc_table); + + return true; +} + +bool regmap_readable_noinc(struct regmap *map, unsigned int reg) +{ + if (map->readable_noinc_reg) + return map->readable_noinc_reg(map->dev, reg); + + if (map->rd_noinc_table) + return regmap_check_range_table(map, reg, map->rd_noinc_table); + + return true; +} + static bool regmap_volatile_range(struct regmap *map, unsigned int reg, size_t num) { unsigned int i; for (i = 0; i < num; i++) - if (!regmap_volatile(map, reg + i)) + if (!regmap_volatile(map, reg + regmap_get_offset(map, i))) return false; return true; } +static void regmap_format_12_20_write(struct regmap *map, + unsigned int reg, unsigned int val) +{ + u8 *out = map->work_buf; + + out[0] = reg >> 4; + out[1] = (reg << 4) | (val >> 16); + out[2] = val >> 8; + out[3] = val; +} + + static void regmap_format_2_6_write(struct regmap *map, unsigned int reg, unsigned int val) { @@ -177,6 +243,16 @@ static void regmap_format_7_9_write(struct regmap *map, *out = cpu_to_be16((reg << 9) | val); } +static void regmap_format_7_17_write(struct regmap *map, + unsigned int reg, unsigned int val) +{ + u8 *out = map->work_buf; + + out[2] = val; + out[1] = val >> 8; + out[0] = (val >> 16) | (reg << 1); +} + static void regmap_format_10_14_write(struct regmap *map, unsigned int reg, unsigned int val) { @@ -196,39 +272,43 @@ static void regmap_format_8(void *buf, unsigned int val, unsigned int shift) static void regmap_format_16_be(void *buf, unsigned int val, unsigned int shift) { - __be16 *b = buf; + put_unaligned_be16(val << shift, buf); +} - b[0] = cpu_to_be16(val << shift); +static void regmap_format_16_le(void *buf, unsigned int val, unsigned int shift) +{ + put_unaligned_le16(val << shift, buf); } static void regmap_format_16_native(void *buf, unsigned int val, unsigned int shift) { - *(u16 *)buf = val << shift; + u16 v = val << shift; + + memcpy(buf, &v, sizeof(v)); } -static void regmap_format_24(void *buf, unsigned int val, unsigned int shift) +static void regmap_format_24_be(void *buf, unsigned int val, unsigned int shift) { - u8 *b = buf; - - val <<= shift; - - b[0] = val >> 16; - b[1] = val >> 8; - b[2] = val; + put_unaligned_be24(val << shift, buf); } static void regmap_format_32_be(void *buf, unsigned int val, unsigned int shift) { - __be32 *b = buf; + put_unaligned_be32(val << shift, buf); +} - b[0] = cpu_to_be32(val << shift); +static void regmap_format_32_le(void *buf, unsigned int val, unsigned int shift) +{ + put_unaligned_le32(val << shift, buf); } static void regmap_format_32_native(void *buf, unsigned int val, unsigned int shift) { - *(u32 *)buf = val << shift; + u32 v = val << shift; + + memcpy(buf, &v, sizeof(v)); } static void regmap_parse_inplace_noop(void *buf) @@ -244,50 +324,119 @@ static unsigned int regmap_parse_8(const void *buf) static unsigned int regmap_parse_16_be(const void *buf) { - const __be16 *b = buf; + return get_unaligned_be16(buf); +} - return be16_to_cpu(b[0]); +static unsigned int regmap_parse_16_le(const void *buf) +{ + return get_unaligned_le16(buf); } static void regmap_parse_16_be_inplace(void *buf) { - __be16 *b = buf; + u16 v = get_unaligned_be16(buf); - b[0] = be16_to_cpu(b[0]); + memcpy(buf, &v, sizeof(v)); } -static unsigned int regmap_parse_16_native(const void *buf) +static void regmap_parse_16_le_inplace(void *buf) { - return *(u16 *)buf; + u16 v = get_unaligned_le16(buf); + + memcpy(buf, &v, sizeof(v)); } -static unsigned int regmap_parse_24(const void *buf) +static unsigned int regmap_parse_16_native(const void *buf) { - const u8 *b = buf; - unsigned int ret = b[2]; - ret |= ((unsigned int)b[1]) << 8; - ret |= ((unsigned int)b[0]) << 16; + u16 v; - return ret; + memcpy(&v, buf, sizeof(v)); + return v; +} + +static unsigned int regmap_parse_24_be(const void *buf) +{ + return get_unaligned_be24(buf); } static unsigned int regmap_parse_32_be(const void *buf) { - const __be32 *b = buf; + return get_unaligned_be32(buf); +} - return be32_to_cpu(b[0]); +static unsigned int regmap_parse_32_le(const void *buf) +{ + return get_unaligned_le32(buf); } static void regmap_parse_32_be_inplace(void *buf) { - __be32 *b = buf; + u32 v = get_unaligned_be32(buf); + + memcpy(buf, &v, sizeof(v)); +} + +static void regmap_parse_32_le_inplace(void *buf) +{ + u32 v = get_unaligned_le32(buf); - b[0] = be32_to_cpu(b[0]); + memcpy(buf, &v, sizeof(v)); } static unsigned int regmap_parse_32_native(const void *buf) { - return *(u32 *)buf; + u32 v; + + memcpy(&v, buf, sizeof(v)); + return v; +} + +static void regmap_lock_hwlock(void *__map) +{ + struct regmap *map = __map; + + hwspin_lock_timeout(map->hwlock, UINT_MAX); +} + +static void regmap_lock_hwlock_irq(void *__map) +{ + struct regmap *map = __map; + + hwspin_lock_timeout_irq(map->hwlock, UINT_MAX); +} + +static void regmap_lock_hwlock_irqsave(void *__map) +{ + struct regmap *map = __map; + + hwspin_lock_timeout_irqsave(map->hwlock, UINT_MAX, + &map->spinlock_flags); +} + +static void regmap_unlock_hwlock(void *__map) +{ + struct regmap *map = __map; + + hwspin_unlock(map->hwlock); +} + +static void regmap_unlock_hwlock_irq(void *__map) +{ + struct regmap *map = __map; + + hwspin_unlock_irq(map->hwlock); +} + +static void regmap_unlock_hwlock_irqrestore(void *__map) +{ + struct regmap *map = __map; + + hwspin_unlock_irqrestore(map->hwlock, &map->spinlock_flags); +} + +static void regmap_lock_unlock_none(void *__map) +{ + } static void regmap_lock_mutex(void *__map) @@ -303,6 +452,7 @@ static void regmap_unlock_mutex(void *__map) } static void regmap_lock_spinlock(void *__map) +__acquires(&map->spinlock) { struct regmap *map = __map; unsigned long flags; @@ -312,11 +462,29 @@ static void regmap_lock_spinlock(void *__map) } static void regmap_unlock_spinlock(void *__map) +__releases(&map->spinlock) { struct regmap *map = __map; spin_unlock_irqrestore(&map->spinlock, map->spinlock_flags); } +static void regmap_lock_raw_spinlock(void *__map) +__acquires(&map->raw_spinlock) +{ + struct regmap *map = __map; + unsigned long flags; + + raw_spin_lock_irqsave(&map->raw_spinlock, flags); + map->raw_spinlock_flags = flags; +} + +static void regmap_unlock_raw_spinlock(void *__map) +__releases(&map->raw_spinlock) +{ + struct regmap *map = __map; + raw_spin_unlock_irqrestore(&map->raw_spinlock, map->raw_spinlock_flags); +} + static void dev_get_regmap_release(struct device *dev, void *res) { /* @@ -334,7 +502,7 @@ static bool _regmap_range_add(struct regmap *map, while (*new) { struct regmap_range_node *this = - container_of(*new, struct regmap_range_node, node); + rb_entry(*new, struct regmap_range_node, node); parent = *new; if (data->range_max < this->range_min) @@ -358,7 +526,7 @@ static struct regmap_range_node *_regmap_range_lookup(struct regmap *map, while (node) { struct regmap_range_node *this = - container_of(node, struct regmap_range_node, node); + rb_entry(node, struct regmap_range_node, node); if (reg < this->range_min) node = node->rb_left; @@ -387,24 +555,131 @@ static void regmap_range_exit(struct regmap *map) kfree(map->selector_work_buf); } -/** - * regmap_init(): Initialise register map - * - * @dev: Device that will be interacted with - * @bus: Bus-specific callbacks to use with device - * @bus_context: Data passed to bus-specific callbacks - * @config: Configuration for register map - * - * The return value will be an ERR_PTR() on error or a valid pointer to - * a struct regmap. This function should generally not be called - * directly, it should be called by bus-specific init functions. - */ -struct regmap *regmap_init(struct device *dev, - const struct regmap_bus *bus, - void *bus_context, - const struct regmap_config *config) +static int regmap_set_name(struct regmap *map, const struct regmap_config *config) +{ + if (config->name) { + const char *name = kstrdup_const(config->name, GFP_KERNEL); + + if (!name) + return -ENOMEM; + + kfree_const(map->name); + map->name = name; + } + + return 0; +} + +int regmap_attach_dev(struct device *dev, struct regmap *map, + const struct regmap_config *config) +{ + struct regmap **m; + int ret; + + map->dev = dev; + + ret = regmap_set_name(map, config); + if (ret) + return ret; + + regmap_debugfs_exit(map); + regmap_debugfs_init(map); + + /* Add a devres resource for dev_get_regmap() */ + m = devres_alloc(dev_get_regmap_release, sizeof(*m), GFP_KERNEL); + if (!m) { + regmap_debugfs_exit(map); + return -ENOMEM; + } + *m = map; + devres_add(dev, m); + + return 0; +} +EXPORT_SYMBOL_GPL(regmap_attach_dev); + +static int dev_get_regmap_match(struct device *dev, void *res, void *data); + +static int regmap_detach_dev(struct device *dev, struct regmap *map) +{ + if (!dev) + return 0; + + return devres_release(dev, dev_get_regmap_release, + dev_get_regmap_match, (void *)map->name); +} + +static enum regmap_endian regmap_get_reg_endian(const struct regmap_bus *bus, + const struct regmap_config *config) { - struct regmap *map, **m; + enum regmap_endian endian; + + /* Retrieve the endianness specification from the regmap config */ + endian = config->reg_format_endian; + + /* If the regmap config specified a non-default value, use that */ + if (endian != REGMAP_ENDIAN_DEFAULT) + return endian; + + /* Retrieve the endianness specification from the bus config */ + if (bus && bus->reg_format_endian_default) + endian = bus->reg_format_endian_default; + + /* If the bus specified a non-default value, use that */ + if (endian != REGMAP_ENDIAN_DEFAULT) + return endian; + + /* Use this if no other value was found */ + return REGMAP_ENDIAN_BIG; +} + +enum regmap_endian regmap_get_val_endian(struct device *dev, + const struct regmap_bus *bus, + const struct regmap_config *config) +{ + struct fwnode_handle *fwnode = dev ? dev_fwnode(dev) : NULL; + enum regmap_endian endian; + + /* Retrieve the endianness specification from the regmap config */ + endian = config->val_format_endian; + + /* If the regmap config specified a non-default value, use that */ + if (endian != REGMAP_ENDIAN_DEFAULT) + return endian; + + /* If the firmware node exist try to get endianness from it */ + if (fwnode_property_read_bool(fwnode, "big-endian")) + endian = REGMAP_ENDIAN_BIG; + else if (fwnode_property_read_bool(fwnode, "little-endian")) + endian = REGMAP_ENDIAN_LITTLE; + else if (fwnode_property_read_bool(fwnode, "native-endian")) + endian = REGMAP_ENDIAN_NATIVE; + + /* If the endianness was specified in fwnode, use that */ + if (endian != REGMAP_ENDIAN_DEFAULT) + return endian; + + /* Retrieve the endianness specification from the bus config */ + if (bus && bus->val_format_endian_default) + endian = bus->val_format_endian_default; + + /* If the bus specified a non-default value, use that */ + if (endian != REGMAP_ENDIAN_DEFAULT) + return endian; + + /* Use this if no other value was found */ + return REGMAP_ENDIAN_BIG; +} +EXPORT_SYMBOL_GPL(regmap_get_val_endian); + +struct regmap *__regmap_init(struct device *dev, + const struct regmap_bus *bus, + void *bus_context, + const struct regmap_config *config, + struct lock_class_key *lock_key, + const char *lock_name) +{ + struct regmap *map; int ret = -EINVAL; enum regmap_endian reg_endian, val_endian; int i, j; @@ -418,81 +693,175 @@ struct regmap *regmap_init(struct device *dev, goto err; } - if (config->lock && config->unlock) { + ret = regmap_set_name(map, config); + if (ret) + goto err_map; + + ret = -EINVAL; /* Later error paths rely on this */ + + if (config->disable_locking) { + map->lock = map->unlock = regmap_lock_unlock_none; + map->can_sleep = config->can_sleep; + regmap_debugfs_disable(map); + } else if (config->lock && config->unlock) { map->lock = config->lock; map->unlock = config->unlock; map->lock_arg = config->lock_arg; + map->can_sleep = config->can_sleep; + } else if (config->use_hwlock) { + map->hwlock = hwspin_lock_request_specific(config->hwlock_id); + if (!map->hwlock) { + ret = -ENXIO; + goto err_name; + } + + switch (config->hwlock_mode) { + case HWLOCK_IRQSTATE: + map->lock = regmap_lock_hwlock_irqsave; + map->unlock = regmap_unlock_hwlock_irqrestore; + break; + case HWLOCK_IRQ: + map->lock = regmap_lock_hwlock_irq; + map->unlock = regmap_unlock_hwlock_irq; + break; + default: + map->lock = regmap_lock_hwlock; + map->unlock = regmap_unlock_hwlock; + break; + } + + map->lock_arg = map; } else { if ((bus && bus->fast_io) || config->fast_io) { - spin_lock_init(&map->spinlock); - map->lock = regmap_lock_spinlock; - map->unlock = regmap_unlock_spinlock; + if (config->use_raw_spinlock) { + raw_spin_lock_init(&map->raw_spinlock); + map->lock = regmap_lock_raw_spinlock; + map->unlock = regmap_unlock_raw_spinlock; + lockdep_set_class_and_name(&map->raw_spinlock, + lock_key, lock_name); + } else { + spin_lock_init(&map->spinlock); + map->lock = regmap_lock_spinlock; + map->unlock = regmap_unlock_spinlock; + lockdep_set_class_and_name(&map->spinlock, + lock_key, lock_name); + } } else { mutex_init(&map->mutex); map->lock = regmap_lock_mutex; map->unlock = regmap_unlock_mutex; + map->can_sleep = true; + lockdep_set_class_and_name(&map->mutex, + lock_key, lock_name); } map->lock_arg = map; + map->lock_key = lock_key; } - map->format.reg_bytes = DIV_ROUND_UP(config->reg_bits, 8); - map->format.pad_bytes = config->pad_bits / 8; - map->format.val_bytes = DIV_ROUND_UP(config->val_bits, 8); - map->format.buf_size = DIV_ROUND_UP(config->reg_bits + - config->val_bits + config->pad_bits, 8); + + /* + * When we write in fast-paths with regmap_bulk_write() don't allocate + * scratch buffers with sleeping allocations. + */ + if ((bus && bus->fast_io) || config->fast_io) + map->alloc_flags = GFP_ATOMIC; + else + map->alloc_flags = GFP_KERNEL; + + map->reg_base = config->reg_base; map->reg_shift = config->pad_bits % 8; + + map->format.pad_bytes = config->pad_bits / 8; + map->format.reg_shift = config->reg_shift; + map->format.reg_bytes = BITS_TO_BYTES(config->reg_bits); + map->format.val_bytes = BITS_TO_BYTES(config->val_bits); + map->format.buf_size = BITS_TO_BYTES(config->reg_bits + config->val_bits + config->pad_bits); if (config->reg_stride) map->reg_stride = config->reg_stride; else map->reg_stride = 1; - map->use_single_rw = config->use_single_rw; + if (is_power_of_2(map->reg_stride)) + map->reg_stride_order = ilog2(map->reg_stride); + else + map->reg_stride_order = -1; + map->use_single_read = config->use_single_read || !(config->read || (bus && bus->read)); + map->use_single_write = config->use_single_write || !(config->write || (bus && bus->write)); + map->can_multi_write = config->can_multi_write && (config->write || (bus && bus->write)); + if (bus) { + map->max_raw_read = bus->max_raw_read; + map->max_raw_write = bus->max_raw_write; + } else if (config->max_raw_read && config->max_raw_write) { + map->max_raw_read = config->max_raw_read; + map->max_raw_write = config->max_raw_write; + } map->dev = dev; map->bus = bus; map->bus_context = bus_context; map->max_register = config->max_register; + map->max_register_is_set = map->max_register ?: config->max_register_is_0; map->wr_table = config->wr_table; map->rd_table = config->rd_table; map->volatile_table = config->volatile_table; map->precious_table = config->precious_table; + map->wr_noinc_table = config->wr_noinc_table; + map->rd_noinc_table = config->rd_noinc_table; map->writeable_reg = config->writeable_reg; map->readable_reg = config->readable_reg; map->volatile_reg = config->volatile_reg; map->precious_reg = config->precious_reg; + map->writeable_noinc_reg = config->writeable_noinc_reg; + map->readable_noinc_reg = config->readable_noinc_reg; map->cache_type = config->cache_type; - map->name = config->name; spin_lock_init(&map->async_lock); INIT_LIST_HEAD(&map->async_list); + INIT_LIST_HEAD(&map->async_free); init_waitqueue_head(&map->async_waitq); - if (config->read_flag_mask || config->write_flag_mask) { + if (config->read_flag_mask || + config->write_flag_mask || + config->zero_flag_mask) { map->read_flag_mask = config->read_flag_mask; map->write_flag_mask = config->write_flag_mask; } else if (bus) { map->read_flag_mask = bus->read_flag_mask; } - if (!bus) { + if (config->read && config->write) { + map->reg_read = _regmap_bus_read; + if (config->reg_update_bits) + map->reg_update_bits = config->reg_update_bits; + + /* Bulk read/write */ + map->read = config->read; + map->write = config->write; + + reg_endian = REGMAP_ENDIAN_NATIVE; + val_endian = REGMAP_ENDIAN_NATIVE; + } else if (!bus) { map->reg_read = config->reg_read; map->reg_write = config->reg_write; + map->reg_update_bits = config->reg_update_bits; + + map->defer_caching = false; + goto skip_format_initialization; + } else if (!bus->read || !bus->write) { + map->reg_read = _regmap_bus_reg_read; + map->reg_write = _regmap_bus_reg_write; + map->reg_update_bits = bus->reg_update_bits; map->defer_caching = false; goto skip_format_initialization; } else { map->reg_read = _regmap_bus_read; - } - - reg_endian = config->reg_format_endian; - if (reg_endian == REGMAP_ENDIAN_DEFAULT) - reg_endian = bus->reg_format_endian_default; - if (reg_endian == REGMAP_ENDIAN_DEFAULT) - reg_endian = REGMAP_ENDIAN_BIG; + map->reg_update_bits = bus->reg_update_bits; + /* Bulk read/write */ + map->read = bus->read; + map->write = bus->write; - val_endian = config->val_format_endian; - if (val_endian == REGMAP_ENDIAN_DEFAULT) - val_endian = bus->val_format_endian_default; - if (val_endian == REGMAP_ENDIAN_DEFAULT) - val_endian = REGMAP_ENDIAN_BIG; + reg_endian = regmap_get_reg_endian(bus, config); + val_endian = regmap_get_val_endian(dev, bus, config); + } switch (config->reg_bits + map->reg_shift) { case 2: @@ -501,7 +870,7 @@ struct regmap *regmap_init(struct device *dev, map->format.format_write = regmap_format_2_6_write; break; default: - goto err_map; + goto err_hwlock; } break; @@ -511,7 +880,7 @@ struct regmap *regmap_init(struct device *dev, map->format.format_write = regmap_format_4_12_write; break; default: - goto err_map; + goto err_hwlock; } break; @@ -520,8 +889,11 @@ struct regmap *regmap_init(struct device *dev, case 9: map->format.format_write = regmap_format_7_9_write; break; + case 17: + map->format.format_write = regmap_format_7_17_write; + break; default: - goto err_map; + goto err_hwlock; } break; @@ -531,7 +903,17 @@ struct regmap *regmap_init(struct device *dev, map->format.format_write = regmap_format_10_14_write; break; default: - goto err_map; + goto err_hwlock; + } + break; + + case 12: + switch (config->val_bits) { + case 20: + map->format.format_write = regmap_format_12_20_write; + break; + default: + goto err_hwlock; } break; @@ -544,18 +926,25 @@ struct regmap *regmap_init(struct device *dev, case REGMAP_ENDIAN_BIG: map->format.format_reg = regmap_format_16_be; break; + case REGMAP_ENDIAN_LITTLE: + map->format.format_reg = regmap_format_16_le; + break; case REGMAP_ENDIAN_NATIVE: map->format.format_reg = regmap_format_16_native; break; default: - goto err_map; + goto err_hwlock; } break; case 24: - if (reg_endian != REGMAP_ENDIAN_BIG) - goto err_map; - map->format.format_reg = regmap_format_24; + switch (reg_endian) { + case REGMAP_ENDIAN_BIG: + map->format.format_reg = regmap_format_24_be; + break; + default: + goto err_hwlock; + } break; case 32: @@ -563,16 +952,19 @@ struct regmap *regmap_init(struct device *dev, case REGMAP_ENDIAN_BIG: map->format.format_reg = regmap_format_32_be; break; + case REGMAP_ENDIAN_LITTLE: + map->format.format_reg = regmap_format_32_le; + break; case REGMAP_ENDIAN_NATIVE: map->format.format_reg = regmap_format_32_native; break; default: - goto err_map; + goto err_hwlock; } break; default: - goto err_map; + goto err_hwlock; } if (val_endian == REGMAP_ENDIAN_NATIVE) @@ -591,19 +983,28 @@ struct regmap *regmap_init(struct device *dev, map->format.parse_val = regmap_parse_16_be; map->format.parse_inplace = regmap_parse_16_be_inplace; break; + case REGMAP_ENDIAN_LITTLE: + map->format.format_val = regmap_format_16_le; + map->format.parse_val = regmap_parse_16_le; + map->format.parse_inplace = regmap_parse_16_le_inplace; + break; case REGMAP_ENDIAN_NATIVE: map->format.format_val = regmap_format_16_native; map->format.parse_val = regmap_parse_16_native; break; default: - goto err_map; + goto err_hwlock; } break; case 24: - if (val_endian != REGMAP_ENDIAN_BIG) - goto err_map; - map->format.format_val = regmap_format_24; - map->format.parse_val = regmap_parse_24; + switch (val_endian) { + case REGMAP_ENDIAN_BIG: + map->format.format_val = regmap_format_24_be; + map->format.parse_val = regmap_parse_24_be; + break; + default: + goto err_hwlock; + } break; case 32: switch (val_endian) { @@ -612,12 +1013,17 @@ struct regmap *regmap_init(struct device *dev, map->format.parse_val = regmap_parse_32_be; map->format.parse_inplace = regmap_parse_32_be_inplace; break; + case REGMAP_ENDIAN_LITTLE: + map->format.format_val = regmap_format_32_le; + map->format.parse_val = regmap_parse_32_le; + map->format.parse_inplace = regmap_parse_32_le_inplace; + break; case REGMAP_ENDIAN_NATIVE: map->format.format_val = regmap_format_32_native; map->format.parse_val = regmap_parse_32_native; break; default: - goto err_map; + goto err_hwlock; } break; } @@ -625,18 +1031,18 @@ struct regmap *regmap_init(struct device *dev, if (map->format.format_write) { if ((reg_endian != REGMAP_ENDIAN_BIG) || (val_endian != REGMAP_ENDIAN_BIG)) - goto err_map; - map->use_single_rw = true; + goto err_hwlock; + map->use_single_write = true; } if (!map->format.format_write && !(map->format.format_reg && map->format.format_val)) - goto err_map; + goto err_hwlock; map->work_buf = kzalloc(map->format.buf_size, GFP_KERNEL); if (map->work_buf == NULL) { ret = -ENOMEM; - goto err_map; + goto err_hwlock; } if (map->format.format_write) { @@ -656,13 +1062,13 @@ skip_format_initialization: /* Sanity check */ if (range_cfg->range_max < range_cfg->range_min) { - dev_err(map->dev, "Invalid range %d: %d < %d\n", i, + dev_err(map->dev, "Invalid range %d: %u < %u\n", i, range_cfg->range_max, range_cfg->range_min); goto err_range; } if (range_cfg->range_max > map->max_register) { - dev_err(map->dev, "Invalid range %d: %d > %d\n", i, + dev_err(map->dev, "Invalid range %d: %u > %u\n", i, range_cfg->range_max, map->max_register); goto err_range; } @@ -682,10 +1088,14 @@ skip_format_initialization: /* Make sure, that this register range has no selector or data window within its boundary */ for (j = 0; j < config->num_ranges; j++) { - unsigned sel_reg = config->ranges[j].selector_reg; - unsigned win_min = config->ranges[j].window_start; - unsigned win_max = win_min + - config->ranges[j].window_len - 1; + unsigned int sel_reg = config->ranges[j].selector_reg; + unsigned int win_min = config->ranges[j].window_start; + unsigned int win_max = win_min + + config->ranges[j].window_len - 1; + + /* Allow data window inside its own virtual range */ + if (j == i) + continue; if (range_cfg->range_min <= sel_reg && sel_reg <= range_cfg->range_max) { @@ -720,7 +1130,7 @@ skip_format_initialization: new->window_start = range_cfg->window_start; new->window_len = range_cfg->window_len; - if (_regmap_range_add(map, new) == false) { + if (!_regmap_range_add(map, new)) { dev_err(map->dev, "Failed to add range %d\n", i); kfree(new); goto err_range; @@ -736,58 +1146,50 @@ skip_format_initialization: } } - regmap_debugfs_init(map, config->name); - ret = regcache_init(map, config); if (ret != 0) goto err_range; - /* Add a devres resource for dev_get_regmap() */ - m = devres_alloc(dev_get_regmap_release, sizeof(*m), GFP_KERNEL); - if (!m) { - ret = -ENOMEM; - goto err_debugfs; + if (dev) { + ret = regmap_attach_dev(dev, map, config); + if (ret != 0) + goto err_regcache; + } else { + regmap_debugfs_init(map); } - *m = map; - devres_add(dev, m); return map; -err_debugfs: - regmap_debugfs_exit(map); +err_regcache: regcache_exit(map); err_range: regmap_range_exit(map); kfree(map->work_buf); +err_hwlock: + if (map->hwlock) + hwspin_lock_free(map->hwlock); +err_name: + kfree_const(map->name); err_map: kfree(map); err: + if (bus && bus->free_on_exit) + kfree(bus); return ERR_PTR(ret); } -EXPORT_SYMBOL_GPL(regmap_init); +EXPORT_SYMBOL_GPL(__regmap_init); static void devm_regmap_release(struct device *dev, void *res) { regmap_exit(*(struct regmap **)res); } -/** - * devm_regmap_init(): Initialise managed register map - * - * @dev: Device that will be interacted with - * @bus: Bus-specific callbacks to use with device - * @bus_context: Data passed to bus-specific callbacks - * @config: Configuration for register map - * - * The return value will be an ERR_PTR() on error or a valid pointer - * to a struct regmap. This function should generally not be called - * directly, it should be called by bus-specific init functions. The - * map will be automatically freed by the device management code. - */ -struct regmap *devm_regmap_init(struct device *dev, - const struct regmap_bus *bus, - void *bus_context, - const struct regmap_config *config) +struct regmap *__devm_regmap_init(struct device *dev, + const struct regmap_bus *bus, + void *bus_context, + const struct regmap_config *config, + struct lock_class_key *lock_key, + const char *lock_name) { struct regmap **ptr, *regmap; @@ -795,7 +1197,8 @@ struct regmap *devm_regmap_init(struct device *dev, if (!ptr) return ERR_PTR(-ENOMEM); - regmap = regmap_init(dev, bus, bus_context, config); + regmap = __regmap_init(dev, bus, bus_context, config, + lock_key, lock_name); if (!IS_ERR(regmap)) { *ptr = regmap; devres_add(dev, ptr); @@ -805,21 +1208,24 @@ struct regmap *devm_regmap_init(struct device *dev, return regmap; } -EXPORT_SYMBOL_GPL(devm_regmap_init); +EXPORT_SYMBOL_GPL(__devm_regmap_init); static void regmap_field_init(struct regmap_field *rm_field, struct regmap *regmap, struct reg_field reg_field) { - int field_bits = reg_field.msb - reg_field.lsb + 1; rm_field->regmap = regmap; rm_field->reg = reg_field.reg; rm_field->shift = reg_field.lsb; - rm_field->mask = ((BIT(field_bits) - 1) << reg_field.lsb); + rm_field->mask = GENMASK(reg_field.msb, reg_field.lsb); + + WARN_ONCE(rm_field->mask == 0, "invalid empty mask defined\n"); + + rm_field->id_size = reg_field.id_size; + rm_field->id_offset = reg_field.id_offset; } /** - * devm_regmap_field_alloc(): Allocate and initialise a register field - * in a register map. + * devm_regmap_field_alloc() - Allocate and initialise a register field. * * @dev: Device that will be interacted with * @regmap: regmap bank in which this register field is located. @@ -844,14 +1250,116 @@ struct regmap_field *devm_regmap_field_alloc(struct device *dev, } EXPORT_SYMBOL_GPL(devm_regmap_field_alloc); + +/** + * regmap_field_bulk_alloc() - Allocate and initialise a bulk register field. + * + * @regmap: regmap bank in which this register field is located. + * @rm_field: regmap register fields within the bank. + * @reg_field: Register fields within the bank. + * @num_fields: Number of register fields. + * + * The return value will be an -ENOMEM on error or zero for success. + * Newly allocated regmap_fields should be freed by calling + * regmap_field_bulk_free() + */ +int regmap_field_bulk_alloc(struct regmap *regmap, + struct regmap_field **rm_field, + const struct reg_field *reg_field, + int num_fields) +{ + struct regmap_field *rf; + int i; + + rf = kcalloc(num_fields, sizeof(*rf), GFP_KERNEL); + if (!rf) + return -ENOMEM; + + for (i = 0; i < num_fields; i++) { + regmap_field_init(&rf[i], regmap, reg_field[i]); + rm_field[i] = &rf[i]; + } + + return 0; +} +EXPORT_SYMBOL_GPL(regmap_field_bulk_alloc); + +/** + * devm_regmap_field_bulk_alloc() - Allocate and initialise a bulk register + * fields. + * + * @dev: Device that will be interacted with + * @regmap: regmap bank in which this register field is located. + * @rm_field: regmap register fields within the bank. + * @reg_field: Register fields within the bank. + * @num_fields: Number of register fields. + * + * The return value will be an -ENOMEM on error or zero for success. + * Newly allocated regmap_fields will be automatically freed by the + * device management code. + */ +int devm_regmap_field_bulk_alloc(struct device *dev, + struct regmap *regmap, + struct regmap_field **rm_field, + const struct reg_field *reg_field, + int num_fields) +{ + struct regmap_field *rf; + int i; + + rf = devm_kcalloc(dev, num_fields, sizeof(*rf), GFP_KERNEL); + if (!rf) + return -ENOMEM; + + for (i = 0; i < num_fields; i++) { + regmap_field_init(&rf[i], regmap, reg_field[i]); + rm_field[i] = &rf[i]; + } + + return 0; +} +EXPORT_SYMBOL_GPL(devm_regmap_field_bulk_alloc); + +/** + * regmap_field_bulk_free() - Free register field allocated using + * regmap_field_bulk_alloc. + * + * @field: regmap fields which should be freed. + */ +void regmap_field_bulk_free(struct regmap_field *field) +{ + kfree(field); +} +EXPORT_SYMBOL_GPL(regmap_field_bulk_free); + +/** + * devm_regmap_field_bulk_free() - Free a bulk register field allocated using + * devm_regmap_field_bulk_alloc. + * + * @dev: Device that will be interacted with + * @field: regmap field which should be freed. + * + * Free register field allocated using devm_regmap_field_bulk_alloc(). Usually + * drivers need not call this function, as the memory allocated via devm + * will be freed as per device-driver life-cycle. + */ +void devm_regmap_field_bulk_free(struct device *dev, + struct regmap_field *field) +{ + devm_kfree(dev, field); +} +EXPORT_SYMBOL_GPL(devm_regmap_field_bulk_free); + /** - * devm_regmap_field_free(): Free register field allocated using - * devm_regmap_field_alloc. Usally drivers need not call this function, - * as the memory allocated via devm will be freed as per device-driver - * life-cyle. + * devm_regmap_field_free() - Free a register field allocated using + * devm_regmap_field_alloc. * * @dev: Device that will be interacted with * @field: regmap field which should be freed. + * + * Free register field allocated using devm_regmap_field_alloc(). Usually + * drivers need not call this function, as the memory allocated via devm + * will be freed as per device-driver life-cyle. */ void devm_regmap_field_free(struct device *dev, struct regmap_field *field) @@ -861,8 +1369,7 @@ void devm_regmap_field_free(struct device *dev, EXPORT_SYMBOL_GPL(devm_regmap_field_free); /** - * regmap_field_alloc(): Allocate and initialise a register field - * in a register map. + * regmap_field_alloc() - Allocate and initialise a register field. * * @regmap: regmap bank in which this register field is located. * @reg_field: Register field with in the bank. @@ -886,7 +1393,8 @@ struct regmap_field *regmap_field_alloc(struct regmap *regmap, EXPORT_SYMBOL_GPL(regmap_field_alloc); /** - * regmap_field_free(): Free register field allocated using regmap_field_alloc + * regmap_field_free() - Free register field allocated using + * regmap_field_alloc. * * @field: regmap field which should be freed. */ @@ -897,7 +1405,7 @@ void regmap_field_free(struct regmap_field *field) EXPORT_SYMBOL_GPL(regmap_field_free); /** - * regmap_reinit_cache(): Reinitialise the current register cache + * regmap_reinit_cache() - Reinitialise the current register cache * * @map: Register map to operate on. * @config: New configuration. Only the cache data will be used. @@ -912,17 +1420,26 @@ EXPORT_SYMBOL_GPL(regmap_field_free); */ int regmap_reinit_cache(struct regmap *map, const struct regmap_config *config) { + int ret; + regcache_exit(map); regmap_debugfs_exit(map); map->max_register = config->max_register; + map->max_register_is_set = map->max_register ?: config->max_register_is_0; map->writeable_reg = config->writeable_reg; map->readable_reg = config->readable_reg; map->volatile_reg = config->volatile_reg; map->precious_reg = config->precious_reg; + map->writeable_noinc_reg = config->writeable_noinc_reg; + map->readable_noinc_reg = config->readable_noinc_reg; map->cache_type = config->cache_type; - regmap_debugfs_init(map, config->name); + ret = regmap_set_name(map, config); + if (ret) + return ret; + + regmap_debugfs_init(map); map->cache_bypass = false; map->cache_only = false; @@ -932,16 +1449,38 @@ int regmap_reinit_cache(struct regmap *map, const struct regmap_config *config) EXPORT_SYMBOL_GPL(regmap_reinit_cache); /** - * regmap_exit(): Free a previously allocated register map + * regmap_exit() - Free a previously allocated register map + * + * @map: Register map to operate on. */ void regmap_exit(struct regmap *map) { + struct regmap_async *async; + + regmap_detach_dev(map->dev, map); regcache_exit(map); + regmap_debugfs_exit(map); regmap_range_exit(map); if (map->bus && map->bus->free_context) map->bus->free_context(map->bus_context); kfree(map->work_buf); + while (!list_empty(&map->async_free)) { + async = list_first_entry_or_null(&map->async_free, + struct regmap_async, + list); + list_del(&async->list); + kfree(async->work_buf); + kfree(async); + } + if (map->hwlock) + hwspin_lock_free(map->hwlock); + if (map->lock == regmap_lock_mutex) + mutex_destroy(&map->mutex); + kfree_const(map->name); + kfree(map->patch); + if (map->bus && map->bus->free_on_exit) + kfree(map->bus); kfree(map); } EXPORT_SYMBOL_GPL(regmap_exit); @@ -956,13 +1495,13 @@ static int dev_get_regmap_match(struct device *dev, void *res, void *data) /* If the user didn't specify a name match any */ if (data) - return (*r)->name == data; + return (*r)->name && !strcmp((*r)->name, data); else return 1; } /** - * dev_get_regmap(): Obtain the regmap (if any) for a device + * dev_get_regmap() - Obtain the regmap (if any) for a device * * @dev: Device to retrieve the map for * @name: Optional name for the register map, usually NULL. @@ -984,6 +1523,19 @@ struct regmap *dev_get_regmap(struct device *dev, const char *name) } EXPORT_SYMBOL_GPL(dev_get_regmap); +/** + * regmap_get_device() - Obtain the device from a regmap + * + * @map: Register map to operate on. + * + * Returns the underlying device that the regmap has been created for. + */ +struct device *regmap_get_device(struct regmap *map) +{ + return map->dev; +} +EXPORT_SYMBOL_GPL(regmap_get_device); + static int _regmap_select_page(struct regmap *map, unsigned int *reg, struct regmap_range_node *range, unsigned int val_num) @@ -1019,7 +1571,7 @@ static int _regmap_select_page(struct regmap *map, unsigned int *reg, ret = _regmap_update_bits(map, range->selector_reg, range->selector_mask, win_page << range->selector_shift, - &page_chg); + &page_chg, false); map->work_buf = orig_work_buf; @@ -1032,12 +1584,38 @@ static int _regmap_select_page(struct regmap *map, unsigned int *reg, return 0; } -int _regmap_raw_write(struct regmap *map, unsigned int reg, - const void *val, size_t val_len, bool async) +static void regmap_set_work_buf_flag_mask(struct regmap *map, int max_bytes, + unsigned long mask) +{ + u8 *buf; + int i; + + if (!mask || !map->work_buf) + return; + + buf = map->work_buf; + + for (i = 0; i < max_bytes; i++) + buf[i] |= (mask >> (8 * i)) & 0xff; +} + +static unsigned int regmap_reg_addr(struct regmap *map, unsigned int reg) +{ + reg += map->reg_base; + + if (map->format.reg_shift > 0) + reg >>= map->format.reg_shift; + else if (map->format.reg_shift < 0) + reg <<= -(map->format.reg_shift); + + return reg; +} + +static int _regmap_raw_write_impl(struct regmap *map, unsigned int reg, + const void *val, size_t val_len, bool noinc) { struct regmap_range_node *range; unsigned long flags; - u8 *u8 = map->work_buf; void *work_val = map->work_buf + map->format.reg_bytes + map->format.pad_bytes; void *buf; @@ -1045,26 +1623,33 @@ int _regmap_raw_write(struct regmap *map, unsigned int reg, size_t len; int i; - WARN_ON(!map->bus); - - /* Check for unwritable registers before we start */ - if (map->writeable_reg) - for (i = 0; i < val_len / map->format.val_bytes; i++) - if (!map->writeable_reg(map->dev, - reg + (i * map->reg_stride))) + /* Check for unwritable or noinc registers in range + * before we start + */ + if (!regmap_writeable_noinc(map, reg)) { + for (i = 0; i < val_len / map->format.val_bytes; i++) { + unsigned int element = + reg + regmap_get_offset(map, i); + if (!regmap_writeable(map, element) || + regmap_writeable_noinc(map, element)) return -EINVAL; + } + } if (!map->cache_bypass && map->format.parse_val) { - unsigned int ival; + unsigned int ival, offset; int val_bytes = map->format.val_bytes; - for (i = 0; i < val_len / val_bytes; i++) { - ival = map->format.parse_val(val + (i * val_bytes)); - ret = regcache_write(map, reg + (i * map->reg_stride), - ival); + + /* Cache the last written value for noinc writes */ + i = noinc ? val_len - val_bytes : 0; + for (; i < val_len; i += val_bytes) { + ival = map->format.parse_val(val + i); + offset = noinc ? 0 : regmap_get_offset(map, i / val_bytes); + ret = regcache_write(map, reg + offset, ival); if (ret) { dev_err(map->dev, "Error in caching of register: %x ret: %d\n", - reg + i, ret); + reg + offset, ret); return ret; } } @@ -1084,8 +1669,9 @@ int _regmap_raw_write(struct regmap *map, unsigned int reg, while (val_num > win_residue) { dev_dbg(map->dev, "Writing window %d/%zu\n", win_residue, val_len / map->format.val_bytes); - ret = _regmap_raw_write(map, reg, val, win_residue * - map->format.val_bytes, async); + ret = _regmap_raw_write_impl(map, reg, val, + win_residue * + map->format.val_bytes, noinc); if (ret != 0) return ret; @@ -1099,80 +1685,105 @@ int _regmap_raw_write(struct regmap *map, unsigned int reg, win_residue = range->window_len - win_offset; } - ret = _regmap_select_page(map, ®, range, val_num); + ret = _regmap_select_page(map, ®, range, noinc ? 1 : val_num); if (ret != 0) return ret; } + reg = regmap_reg_addr(map, reg); map->format.format_reg(map->work_buf, reg, map->reg_shift); + regmap_set_work_buf_flag_mask(map, map->format.reg_bytes, + map->write_flag_mask); - u8[0] |= map->write_flag_mask; + /* + * Essentially all I/O mechanisms will be faster with a single + * buffer to write. Since register syncs often generate raw + * writes of single registers optimise that case. + */ + if (val != work_val && val_len == map->format.val_bytes) { + memcpy(work_val, val, map->format.val_bytes); + val = work_val; + } - if (async && map->bus->async_write) { - struct regmap_async *async = map->bus->async_alloc(); - if (!async) - return -ENOMEM; + if (map->async && map->bus && map->bus->async_write) { + struct regmap_async *async; - trace_regmap_async_write_start(map->dev, reg, val_len); + trace_regmap_async_write_start(map, reg, val_len); - async->work_buf = kzalloc(map->format.buf_size, - GFP_KERNEL | GFP_DMA); - if (!async->work_buf) { - kfree(async); - return -ENOMEM; + spin_lock_irqsave(&map->async_lock, flags); + async = list_first_entry_or_null(&map->async_free, + struct regmap_async, + list); + if (async) + list_del(&async->list); + spin_unlock_irqrestore(&map->async_lock, flags); + + if (!async) { + async = map->bus->async_alloc(); + if (!async) + return -ENOMEM; + + async->work_buf = kzalloc(map->format.buf_size, + GFP_KERNEL | GFP_DMA); + if (!async->work_buf) { + kfree(async); + return -ENOMEM; + } } - INIT_WORK(&async->cleanup, async_cleanup); async->map = map; /* If the caller supplied the value we can use it safely. */ memcpy(async->work_buf, map->work_buf, map->format.pad_bytes + map->format.reg_bytes + map->format.val_bytes); - if (val == work_val) - val = async->work_buf + map->format.pad_bytes + - map->format.reg_bytes; spin_lock_irqsave(&map->async_lock, flags); list_add_tail(&async->list, &map->async_list); spin_unlock_irqrestore(&map->async_lock, flags); - ret = map->bus->async_write(map->bus_context, async->work_buf, - map->format.reg_bytes + - map->format.pad_bytes, - val, val_len, async); + if (val != work_val) + ret = map->bus->async_write(map->bus_context, + async->work_buf, + map->format.reg_bytes + + map->format.pad_bytes, + val, val_len, async); + else + ret = map->bus->async_write(map->bus_context, + async->work_buf, + map->format.reg_bytes + + map->format.pad_bytes + + val_len, NULL, 0, async); if (ret != 0) { dev_err(map->dev, "Failed to schedule write: %d\n", ret); spin_lock_irqsave(&map->async_lock, flags); - list_del(&async->list); + list_move(&async->list, &map->async_free); spin_unlock_irqrestore(&map->async_lock, flags); - - kfree(async->work_buf); - kfree(async); } return ret; } - trace_regmap_hw_write_start(map->dev, reg, - val_len / map->format.val_bytes); + trace_regmap_hw_write_start(map, reg, val_len / map->format.val_bytes); /* If we're doing a single register write we can probably just * send the work_buf directly, otherwise try to do a gather * write. */ if (val == work_val) - ret = map->bus->write(map->bus_context, map->work_buf, - map->format.reg_bytes + - map->format.pad_bytes + - val_len); - else if (map->bus->gather_write) + ret = map->write(map->bus_context, map->work_buf, + map->format.reg_bytes + + map->format.pad_bytes + + val_len); + else if (map->bus && map->bus->gather_write) ret = map->bus->gather_write(map->bus_context, map->work_buf, map->format.reg_bytes + map->format.pad_bytes, val, val_len); + else + ret = -ENOTSUPP; /* If that didn't work fall back on linearising by hand. */ if (ret == -ENOTSUPP) { @@ -1184,13 +1795,18 @@ int _regmap_raw_write(struct regmap *map, unsigned int reg, memcpy(buf, map->work_buf, map->format.reg_bytes); memcpy(buf + map->format.reg_bytes + map->format.pad_bytes, val, val_len); - ret = map->bus->write(map->bus_context, buf, len); + ret = map->write(map->bus_context, buf, len); kfree(buf); + } else if (ret != 0 && !map->cache_bypass && map->format.parse_val) { + /* regcache_drop_region() takes lock that we already have, + * thus call map->cache_ops->drop() directly + */ + if (map->cache_ops && map->cache_ops->drop) + map->cache_ops->drop(map, reg, reg + 1); } - trace_regmap_hw_write_done(map->dev, reg, - val_len / map->format.val_bytes); + trace_regmap_hw_write_done(map, reg, val_len / map->format.val_bytes); return ret; } @@ -1202,10 +1818,32 @@ int _regmap_raw_write(struct regmap *map, unsigned int reg, */ bool regmap_can_raw_write(struct regmap *map) { - return map->bus && map->format.format_val && map->format.format_reg; + return map->write && map->format.format_val && map->format.format_reg; } EXPORT_SYMBOL_GPL(regmap_can_raw_write); +/** + * regmap_get_raw_read_max - Get the maximum size we can read + * + * @map: Map to check. + */ +size_t regmap_get_raw_read_max(struct regmap *map) +{ + return map->max_raw_read; +} +EXPORT_SYMBOL_GPL(regmap_get_raw_read_max); + +/** + * regmap_get_raw_write_max - Get the maximum size we can read + * + * @map: Map to check. + */ +size_t regmap_get_raw_write_max(struct regmap *map) +{ + return map->max_raw_write; +} +EXPORT_SYMBOL_GPL(regmap_get_raw_write_max); + static int _regmap_bus_formatted_write(void *context, unsigned int reg, unsigned int val) { @@ -1213,7 +1851,7 @@ static int _regmap_bus_formatted_write(void *context, unsigned int reg, struct regmap_range_node *range; struct regmap *map = context; - WARN_ON(!map->bus || !map->format.format_write); + WARN_ON(!map->format.format_write); range = _regmap_range_lookup(map, reg); if (range) { @@ -1222,37 +1860,56 @@ static int _regmap_bus_formatted_write(void *context, unsigned int reg, return ret; } + reg = regmap_reg_addr(map, reg); map->format.format_write(map, reg, val); - trace_regmap_hw_write_start(map->dev, reg, 1); + trace_regmap_hw_write_start(map, reg, 1); - ret = map->bus->write(map->bus_context, map->work_buf, - map->format.buf_size); + ret = map->write(map->bus_context, map->work_buf, map->format.buf_size); - trace_regmap_hw_write_done(map->dev, reg, 1); + trace_regmap_hw_write_done(map, reg, 1); return ret; } +static int _regmap_bus_reg_write(void *context, unsigned int reg, + unsigned int val) +{ + struct regmap *map = context; + struct regmap_range_node *range; + int ret; + + range = _regmap_range_lookup(map, reg); + if (range) { + ret = _regmap_select_page(map, ®, range, 1); + if (ret != 0) + return ret; + } + + reg = regmap_reg_addr(map, reg); + return map->bus->reg_write(map->bus_context, reg, val); +} + static int _regmap_bus_raw_write(void *context, unsigned int reg, unsigned int val) { struct regmap *map = context; - WARN_ON(!map->bus || !map->format.format_val); + WARN_ON(!map->format.format_val); map->format.format_val(map->work_buf + map->format.reg_bytes + map->format.pad_bytes, val, 0); - return _regmap_raw_write(map, reg, - map->work_buf + - map->format.reg_bytes + - map->format.pad_bytes, - map->format.val_bytes, false); + return _regmap_raw_write_impl(map, reg, + map->work_buf + + map->format.reg_bytes + + map->format.pad_bytes, + map->format.val_bytes, + false); } static inline void *_regmap_map_get_context(struct regmap *map) { - return (map->bus) ? map : map->bus_context; + return (map->bus || (!map->bus && map->read)) ? map : map->bus_context; } int _regmap_write(struct regmap *map, unsigned int reg, @@ -1261,6 +1918,9 @@ int _regmap_write(struct regmap *map, unsigned int reg, int ret; void *context = _regmap_map_get_context(map); + if (!regmap_writeable(map, reg)) + return -EIO; + if (!map->cache_bypass && !map->defer_caching) { ret = regcache_write(map, reg, val); if (ret != 0) @@ -1271,18 +1931,19 @@ int _regmap_write(struct regmap *map, unsigned int reg, } } -#ifdef LOG_DEVICE - if (strcmp(dev_name(map->dev), LOG_DEVICE) == 0) - dev_info(map->dev, "%x <= %x\n", reg, val); -#endif + ret = map->reg_write(context, reg, val); + if (ret == 0) { + if (regmap_should_log(map)) + dev_info(map->dev, "%x <= %x\n", reg, val); - trace_regmap_reg_write(map->dev, reg, val); + trace_regmap_reg_write(map, reg, val); + } - return map->reg_write(context, reg, val); + return ret; } /** - * regmap_write(): Write a value to a single register + * regmap_write() - Write a value to a single register * * @map: Register map to write to * @reg: Register to write to @@ -1295,7 +1956,7 @@ int regmap_write(struct regmap *map, unsigned int reg, unsigned int val) { int ret; - if (reg % map->reg_stride) + if (!IS_ALIGNED(reg, map->reg_stride)) return -EINVAL; map->lock(map->lock_arg); @@ -1309,7 +1970,76 @@ int regmap_write(struct regmap *map, unsigned int reg, unsigned int val) EXPORT_SYMBOL_GPL(regmap_write); /** - * regmap_raw_write(): Write raw values to one or more registers + * regmap_write_async() - Write a value to a single register asynchronously + * + * @map: Register map to write to + * @reg: Register to write to + * @val: Value to be written + * + * A value of zero will be returned on success, a negative errno will + * be returned in error cases. + */ +int regmap_write_async(struct regmap *map, unsigned int reg, unsigned int val) +{ + int ret; + + if (!IS_ALIGNED(reg, map->reg_stride)) + return -EINVAL; + + map->lock(map->lock_arg); + + map->async = true; + + ret = _regmap_write(map, reg, val); + + map->async = false; + + map->unlock(map->lock_arg); + + return ret; +} +EXPORT_SYMBOL_GPL(regmap_write_async); + +int _regmap_raw_write(struct regmap *map, unsigned int reg, + const void *val, size_t val_len, bool noinc) +{ + size_t val_bytes = map->format.val_bytes; + size_t val_count = val_len / val_bytes; + size_t chunk_count, chunk_bytes; + size_t chunk_regs = val_count; + int ret, i; + + if (!val_count) + return -EINVAL; + + if (map->use_single_write) + chunk_regs = 1; + else if (map->max_raw_write && val_len > map->max_raw_write) + chunk_regs = map->max_raw_write / val_bytes; + + chunk_count = val_count / chunk_regs; + chunk_bytes = chunk_regs * val_bytes; + + /* Write as many bytes as possible with chunk_size */ + for (i = 0; i < chunk_count; i++) { + ret = _regmap_raw_write_impl(map, reg, val, chunk_bytes, noinc); + if (ret) + return ret; + + reg += regmap_get_offset(map, chunk_regs); + val += chunk_bytes; + val_len -= chunk_bytes; + } + + /* Write remaining bytes */ + if (val_len) + ret = _regmap_raw_write_impl(map, reg, val, val_len, noinc); + + return ret; +} + +/** + * regmap_raw_write() - Write raw values to one or more registers * * @map: Register map to write to * @reg: Initial register to write to @@ -1344,24 +2074,240 @@ int regmap_raw_write(struct regmap *map, unsigned int reg, } EXPORT_SYMBOL_GPL(regmap_raw_write); +static int regmap_noinc_readwrite(struct regmap *map, unsigned int reg, + void *val, unsigned int val_len, bool write) +{ + size_t val_bytes = map->format.val_bytes; + size_t val_count = val_len / val_bytes; + unsigned int lastval; + u8 *u8p; + u16 *u16p; + u32 *u32p; + int ret; + int i; + + switch (val_bytes) { + case 1: + u8p = val; + if (write) + lastval = (unsigned int)u8p[val_count - 1]; + break; + case 2: + u16p = val; + if (write) + lastval = (unsigned int)u16p[val_count - 1]; + break; + case 4: + u32p = val; + if (write) + lastval = (unsigned int)u32p[val_count - 1]; + break; + default: + return -EINVAL; + } + + /* + * Update the cache with the last value we write, the rest is just + * gone down in the hardware FIFO. We can't cache FIFOs. This makes + * sure a single read from the cache will work. + */ + if (write) { + if (!map->cache_bypass && !map->defer_caching) { + ret = regcache_write(map, reg, lastval); + if (ret != 0) + return ret; + if (map->cache_only) { + map->cache_dirty = true; + return 0; + } + } + ret = map->bus->reg_noinc_write(map->bus_context, reg, val, val_count); + } else { + ret = map->bus->reg_noinc_read(map->bus_context, reg, val, val_count); + } + + if (!ret && regmap_should_log(map)) { + dev_info(map->dev, "%x %s [", reg, write ? "<=" : "=>"); + for (i = 0; i < val_count; i++) { + switch (val_bytes) { + case 1: + pr_cont("%x", u8p[i]); + break; + case 2: + pr_cont("%x", u16p[i]); + break; + case 4: + pr_cont("%x", u32p[i]); + break; + default: + break; + } + if (i == (val_count - 1)) + pr_cont("]\n"); + else + pr_cont(","); + } + } + + return 0; +} + +/** + * regmap_noinc_write(): Write data to a register without incrementing the + * register number + * + * @map: Register map to write to + * @reg: Register to write to + * @val: Pointer to data buffer + * @val_len: Length of output buffer in bytes. + * + * The regmap API usually assumes that bulk bus write operations will write a + * range of registers. Some devices have certain registers for which a write + * operation can write to an internal FIFO. + * + * The target register must be volatile but registers after it can be + * completely unrelated cacheable registers. + * + * This will attempt multiple writes as required to write val_len bytes. + * + * A value of zero will be returned on success, a negative errno will be + * returned in error cases. + */ +int regmap_noinc_write(struct regmap *map, unsigned int reg, + const void *val, size_t val_len) +{ + size_t write_len; + int ret; + + if (!map->write && !(map->bus && map->bus->reg_noinc_write)) + return -EINVAL; + if (val_len % map->format.val_bytes) + return -EINVAL; + if (!IS_ALIGNED(reg, map->reg_stride)) + return -EINVAL; + if (val_len == 0) + return -EINVAL; + + map->lock(map->lock_arg); + + if (!regmap_volatile(map, reg) || !regmap_writeable_noinc(map, reg)) { + ret = -EINVAL; + goto out_unlock; + } + + /* + * Use the accelerated operation if we can. The val drops the const + * typing in order to facilitate code reuse in regmap_noinc_readwrite(). + */ + if (map->bus->reg_noinc_write) { + ret = regmap_noinc_readwrite(map, reg, (void *)val, val_len, true); + goto out_unlock; + } + + while (val_len) { + if (map->max_raw_write && map->max_raw_write < val_len) + write_len = map->max_raw_write; + else + write_len = val_len; + ret = _regmap_raw_write(map, reg, val, write_len, true); + if (ret) + goto out_unlock; + val = ((u8 *)val) + write_len; + val_len -= write_len; + } + +out_unlock: + map->unlock(map->lock_arg); + return ret; +} +EXPORT_SYMBOL_GPL(regmap_noinc_write); + /** - * regmap_field_write(): Write a value to a single register field + * regmap_field_update_bits_base() - Perform a read/modify/write cycle a + * register field. * * @field: Register field to write to + * @mask: Bitmask to change * @val: Value to be written + * @change: Boolean indicating if a write was done + * @async: Boolean indicating asynchronously + * @force: Boolean indicating use force update + * + * Perform a read/modify/write cycle on the register field with change, + * async, force option. * * A value of zero will be returned on success, a negative errno will * be returned in error cases. */ -int regmap_field_write(struct regmap_field *field, unsigned int val) +int regmap_field_update_bits_base(struct regmap_field *field, + unsigned int mask, unsigned int val, + bool *change, bool async, bool force) { - return regmap_update_bits(field->regmap, field->reg, - field->mask, val << field->shift); + mask = (mask << field->shift) & field->mask; + + return regmap_update_bits_base(field->regmap, field->reg, + mask, val << field->shift, + change, async, force); } -EXPORT_SYMBOL_GPL(regmap_field_write); +EXPORT_SYMBOL_GPL(regmap_field_update_bits_base); -/* - * regmap_bulk_write(): Write multiple registers to the device +/** + * regmap_field_test_bits() - Check if all specified bits are set in a + * register field. + * + * @field: Register field to operate on + * @bits: Bits to test + * + * Returns negative errno if the underlying regmap_field_read() fails, + * 0 if at least one of the tested bits is not set and 1 if all tested + * bits are set. + */ +int regmap_field_test_bits(struct regmap_field *field, unsigned int bits) +{ + unsigned int val; + int ret; + + ret = regmap_field_read(field, &val); + if (ret) + return ret; + + return (val & bits) == bits; +} +EXPORT_SYMBOL_GPL(regmap_field_test_bits); + +/** + * regmap_fields_update_bits_base() - Perform a read/modify/write cycle a + * register field with port ID + * + * @field: Register field to write to + * @id: port ID + * @mask: Bitmask to change + * @val: Value to be written + * @change: Boolean indicating if a write was done + * @async: Boolean indicating asynchronously + * @force: Boolean indicating use force update + * + * A value of zero will be returned on success, a negative errno will + * be returned in error cases. + */ +int regmap_fields_update_bits_base(struct regmap_field *field, unsigned int id, + unsigned int mask, unsigned int val, + bool *change, bool async, bool force) +{ + if (id >= field->id_size) + return -EINVAL; + + mask = (mask << field->shift) & field->mask; + + return regmap_update_bits_base(field->regmap, + field->reg + (field->id_offset * id), + mask, val << field->shift, + change, async, force); +} +EXPORT_SYMBOL_GPL(regmap_fields_update_bits_base); + +/** + * regmap_bulk_write() - Write multiple registers to the device * * @map: Register map to write to * @reg: First register to be write from @@ -1379,60 +2325,366 @@ int regmap_bulk_write(struct regmap *map, unsigned int reg, const void *val, { int ret = 0, i; size_t val_bytes = map->format.val_bytes; - void *wval; - if (!map->bus) - return -EINVAL; - if (!map->format.parse_inplace) - return -EINVAL; - if (reg % map->reg_stride) + if (!IS_ALIGNED(reg, map->reg_stride)) return -EINVAL; - map->lock(map->lock_arg); + /* + * Some devices don't support bulk write, for them we have a series of + * single write operations. + */ + if (!map->write || !map->format.parse_inplace) { + map->lock(map->lock_arg); + for (i = 0; i < val_count; i++) { + unsigned int ival; - /* No formatting is require if val_byte is 1 */ - if (val_bytes == 1) { - wval = (void *)val; - } else { - wval = kmemdup(val, val_count * val_bytes, GFP_KERNEL); - if (!wval) { - ret = -ENOMEM; - dev_err(map->dev, "Error in memory allocation\n"); - goto out; + switch (val_bytes) { + case 1: + ival = *(u8 *)(val + (i * val_bytes)); + break; + case 2: + ival = *(u16 *)(val + (i * val_bytes)); + break; + case 4: + ival = *(u32 *)(val + (i * val_bytes)); + break; + default: + ret = -EINVAL; + goto out; + } + + ret = _regmap_write(map, + reg + regmap_get_offset(map, i), + ival); + if (ret != 0) + goto out; } +out: + map->unlock(map->lock_arg); + } else { + void *wval; + + wval = kmemdup_array(val, val_count, val_bytes, map->alloc_flags); + if (!wval) + return -ENOMEM; + for (i = 0; i < val_count * val_bytes; i += val_bytes) map->format.parse_inplace(wval + i); + + ret = regmap_raw_write(map, reg, wval, val_bytes * val_count); + + kfree(wval); + } + + if (!ret) + trace_regmap_bulk_write(map, reg, val, val_bytes * val_count); + + return ret; +} +EXPORT_SYMBOL_GPL(regmap_bulk_write); + +/* + * _regmap_raw_multi_reg_write() + * + * the (register,newvalue) pairs in regs have not been formatted, but + * they are all in the same page and have been changed to being page + * relative. The page register has been written if that was necessary. + */ +static int _regmap_raw_multi_reg_write(struct regmap *map, + const struct reg_sequence *regs, + size_t num_regs) +{ + int ret; + void *buf; + int i; + u8 *u8; + size_t val_bytes = map->format.val_bytes; + size_t reg_bytes = map->format.reg_bytes; + size_t pad_bytes = map->format.pad_bytes; + size_t pair_size = reg_bytes + pad_bytes + val_bytes; + size_t len = pair_size * num_regs; + + if (!len) + return -EINVAL; + + buf = kzalloc(len, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + /* We have to linearise by hand. */ + + u8 = buf; + + for (i = 0; i < num_regs; i++) { + unsigned int reg = regs[i].reg; + unsigned int val = regs[i].def; + trace_regmap_hw_write_start(map, reg, 1); + reg = regmap_reg_addr(map, reg); + map->format.format_reg(u8, reg, map->reg_shift); + u8 += reg_bytes + pad_bytes; + map->format.format_val(u8, val, 0); + u8 += val_bytes; + } + u8 = buf; + *u8 |= map->write_flag_mask; + + ret = map->write(map->bus_context, buf, len); + + kfree(buf); + + for (i = 0; i < num_regs; i++) { + int reg = regs[i].reg; + trace_regmap_hw_write_done(map, reg, 1); } + return ret; +} + +static unsigned int _regmap_register_page(struct regmap *map, + unsigned int reg, + struct regmap_range_node *range) +{ + unsigned int win_page = (reg - range->range_min) / range->window_len; + + return win_page; +} + +static int _regmap_range_multi_paged_reg_write(struct regmap *map, + struct reg_sequence *regs, + size_t num_regs) +{ + int ret; + int i, n; + struct reg_sequence *base; + unsigned int this_page = 0; + unsigned int page_change = 0; /* - * Some devices does not support bulk write, for - * them we have a series of single write operations. + * the set of registers are not neccessarily in order, but + * since the order of write must be preserved this algorithm + * chops the set each time the page changes. This also applies + * if there is a delay required at any point in the sequence. */ - if (map->use_single_rw) { - for (i = 0; i < val_count; i++) { - ret = regmap_raw_write(map, - reg + (i * map->reg_stride), - val + (i * val_bytes), - val_bytes); + base = regs; + for (i = 0, n = 0; i < num_regs; i++, n++) { + unsigned int reg = regs[i].reg; + struct regmap_range_node *range; + + range = _regmap_range_lookup(map, reg); + if (range) { + unsigned int win_page = _regmap_register_page(map, reg, + range); + + if (i == 0) + this_page = win_page; + if (win_page != this_page) { + this_page = win_page; + page_change = 1; + } + } + + /* If we have both a page change and a delay make sure to + * write the regs and apply the delay before we change the + * page. + */ + + if (page_change || regs[i].delay_us) { + + /* For situations where the first write requires + * a delay we need to make sure we don't call + * raw_multi_reg_write with n=0 + * This can't occur with page breaks as we + * never write on the first iteration + */ + if (regs[i].delay_us && i == 0) + n = 1; + + ret = _regmap_raw_multi_reg_write(map, base, n); + if (ret != 0) + return ret; + + if (regs[i].delay_us) { + if (map->can_sleep) + fsleep(regs[i].delay_us); + else + udelay(regs[i].delay_us); + } + + base += n; + n = 0; + + if (page_change) { + ret = _regmap_select_page(map, + &base[n].reg, + range, 1); + if (ret != 0) + return ret; + + page_change = 0; + } + + } + + } + if (n > 0) + return _regmap_raw_multi_reg_write(map, base, n); + return 0; +} + +static int _regmap_multi_reg_write(struct regmap *map, + const struct reg_sequence *regs, + size_t num_regs) +{ + int i; + int ret; + + if (!map->can_multi_write) { + for (i = 0; i < num_regs; i++) { + ret = _regmap_write(map, regs[i].reg, regs[i].def); if (ret != 0) return ret; + + if (regs[i].delay_us) { + if (map->can_sleep) + fsleep(regs[i].delay_us); + else + udelay(regs[i].delay_us); + } } - } else { - ret = _regmap_raw_write(map, reg, wval, val_bytes * val_count, - false); + return 0; } - if (val_bytes != 1) - kfree(wval); + if (!map->format.parse_inplace) + return -EINVAL; + + if (map->writeable_reg) + for (i = 0; i < num_regs; i++) { + int reg = regs[i].reg; + if (!map->writeable_reg(map->dev, reg)) + return -EINVAL; + if (!IS_ALIGNED(reg, map->reg_stride)) + return -EINVAL; + } + + if (!map->cache_bypass) { + for (i = 0; i < num_regs; i++) { + unsigned int val = regs[i].def; + unsigned int reg = regs[i].reg; + ret = regcache_write(map, reg, val); + if (ret) { + dev_err(map->dev, + "Error in caching of register: %x ret: %d\n", + reg, ret); + return ret; + } + } + if (map->cache_only) { + map->cache_dirty = true; + return 0; + } + } + + WARN_ON(!map->bus); + + for (i = 0; i < num_regs; i++) { + unsigned int reg = regs[i].reg; + struct regmap_range_node *range; + + /* Coalesce all the writes between a page break or a delay + * in a sequence + */ + range = _regmap_range_lookup(map, reg); + if (range || regs[i].delay_us) { + size_t len = sizeof(struct reg_sequence)*num_regs; + struct reg_sequence *base = kmemdup(regs, len, + GFP_KERNEL); + if (!base) + return -ENOMEM; + ret = _regmap_range_multi_paged_reg_write(map, base, + num_regs); + kfree(base); + + return ret; + } + } + return _regmap_raw_multi_reg_write(map, regs, num_regs); +} + +/** + * regmap_multi_reg_write() - Write multiple registers to the device + * + * @map: Register map to write to + * @regs: Array of structures containing register,value to be written + * @num_regs: Number of registers to write + * + * Write multiple registers to the device where the set of register, value + * pairs are supplied in any order, possibly not all in a single range. + * + * The 'normal' block write mode will send ultimately send data on the + * target bus as R,V1,V2,V3,..,Vn where successively higher registers are + * addressed. However, this alternative block multi write mode will send + * the data as R1,V1,R2,V2,..,Rn,Vn on the target bus. The target device + * must of course support the mode. + * + * A value of zero will be returned on success, a negative errno will be + * returned in error cases. + */ +int regmap_multi_reg_write(struct regmap *map, const struct reg_sequence *regs, + int num_regs) +{ + int ret; + + map->lock(map->lock_arg); + + ret = _regmap_multi_reg_write(map, regs, num_regs); -out: map->unlock(map->lock_arg); + return ret; } -EXPORT_SYMBOL_GPL(regmap_bulk_write); +EXPORT_SYMBOL_GPL(regmap_multi_reg_write); /** - * regmap_raw_write_async(): Write raw values to one or more registers - * asynchronously + * regmap_multi_reg_write_bypassed() - Write multiple registers to the + * device but not the cache + * + * @map: Register map to write to + * @regs: Array of structures containing register,value to be written + * @num_regs: Number of registers to write + * + * Write multiple registers to the device but not the cache where the set + * of register are supplied in any order. + * + * This function is intended to be used for writing a large block of data + * atomically to the device in single transfer for those I2C client devices + * that implement this alternative block write mode. + * + * A value of zero will be returned on success, a negative errno will + * be returned in error cases. + */ +int regmap_multi_reg_write_bypassed(struct regmap *map, + const struct reg_sequence *regs, + int num_regs) +{ + int ret; + bool bypass; + + map->lock(map->lock_arg); + + bypass = map->cache_bypass; + map->cache_bypass = true; + + ret = _regmap_multi_reg_write(map, regs, num_regs); + + map->cache_bypass = bypass; + + map->unlock(map->lock_arg); + + return ret; +} +EXPORT_SYMBOL_GPL(regmap_multi_reg_write_bypassed); + +/** + * regmap_raw_write_async() - Write raw values to one or more registers + * asynchronously * * @map: Register map to write to * @reg: Initial register to write to @@ -1459,12 +2711,16 @@ int regmap_raw_write_async(struct regmap *map, unsigned int reg, if (val_len % map->format.val_bytes) return -EINVAL; - if (reg % map->reg_stride) + if (!IS_ALIGNED(reg, map->reg_stride)) return -EINVAL; map->lock(map->lock_arg); - ret = _regmap_raw_write(map, reg, val, val_len, true); + map->async = true; + + ret = _regmap_raw_write(map, reg, val, val_len, false); + + map->async = false; map->unlock(map->lock_arg); @@ -1473,43 +2729,53 @@ int regmap_raw_write_async(struct regmap *map, unsigned int reg, EXPORT_SYMBOL_GPL(regmap_raw_write_async); static int _regmap_raw_read(struct regmap *map, unsigned int reg, void *val, - unsigned int val_len) + unsigned int val_len, bool noinc) { struct regmap_range_node *range; - u8 *u8 = map->work_buf; int ret; - WARN_ON(!map->bus); + if (!map->read) + return -EINVAL; range = _regmap_range_lookup(map, reg); if (range) { ret = _regmap_select_page(map, ®, range, - val_len / map->format.val_bytes); + noinc ? 1 : val_len / map->format.val_bytes); if (ret != 0) return ret; } + reg = regmap_reg_addr(map, reg); map->format.format_reg(map->work_buf, reg, map->reg_shift); + regmap_set_work_buf_flag_mask(map, map->format.reg_bytes, + map->read_flag_mask); + trace_regmap_hw_read_start(map, reg, val_len / map->format.val_bytes); - /* - * Some buses or devices flag reads by setting the high bits in the - * register addresss; since it's always the high bits for all - * current formats we can do this here rather than in - * formatting. This may break if we get interesting formats. - */ - u8[0] |= map->read_flag_mask; + ret = map->read(map->bus_context, map->work_buf, + map->format.reg_bytes + map->format.pad_bytes, + val, val_len); - trace_regmap_hw_read_start(map->dev, reg, - val_len / map->format.val_bytes); + trace_regmap_hw_read_done(map, reg, val_len / map->format.val_bytes); - ret = map->bus->read(map->bus_context, map->work_buf, - map->format.reg_bytes + map->format.pad_bytes, - val, val_len); + return ret; +} - trace_regmap_hw_read_done(map->dev, reg, - val_len / map->format.val_bytes); +static int _regmap_bus_reg_read(void *context, unsigned int reg, + unsigned int *val) +{ + struct regmap *map = context; + struct regmap_range_node *range; + int ret; - return ret; + range = _regmap_range_lookup(map, reg); + if (range) { + ret = _regmap_select_page(map, ®, range, 1); + if (ret != 0) + return ret; + } + + reg = regmap_reg_addr(map, reg); + return map->bus->reg_read(map->bus_context, reg, val); } static int _regmap_bus_read(void *context, unsigned int reg, @@ -1517,13 +2783,15 @@ static int _regmap_bus_read(void *context, unsigned int reg, { int ret; struct regmap *map = context; + void *work_val = map->work_buf + map->format.reg_bytes + + map->format.pad_bytes; if (!map->format.parse_val) return -EINVAL; - ret = _regmap_raw_read(map, reg, map->work_buf, map->format.val_bytes); + ret = _regmap_raw_read(map, reg, work_val, map->format.val_bytes, false); if (ret == 0) - *val = map->format.parse_val(map->work_buf); + *val = map->format.parse_val(work_val); return ret; } @@ -1534,8 +2802,6 @@ static int _regmap_read(struct regmap *map, unsigned int reg, int ret; void *context = _regmap_map_get_context(map); - WARN_ON(!map->reg_read); - if (!map->cache_bypass) { ret = regcache_read(map, reg, val); if (ret == 0) @@ -1545,14 +2811,15 @@ static int _regmap_read(struct regmap *map, unsigned int reg, if (map->cache_only) return -EBUSY; + if (!regmap_readable(map, reg)) + return -EIO; + ret = map->reg_read(context, reg, val); if (ret == 0) { -#ifdef LOG_DEVICE - if (strcmp(dev_name(map->dev), LOG_DEVICE) == 0) + if (regmap_should_log(map)) dev_info(map->dev, "%x => %x\n", reg, *val); -#endif - trace_regmap_reg_read(map->dev, reg, *val); + trace_regmap_reg_read(map, reg, *val); if (!map->cache_bypass) regcache_write(map, reg, *val); @@ -1562,9 +2829,9 @@ static int _regmap_read(struct regmap *map, unsigned int reg, } /** - * regmap_read(): Read a value from a single register + * regmap_read() - Read a value from a single register * - * @map: Register map to write to + * @map: Register map to read from * @reg: Register to be read from * @val: Pointer to store read value * @@ -1575,7 +2842,7 @@ int regmap_read(struct regmap *map, unsigned int reg, unsigned int *val) { int ret; - if (reg % map->reg_stride) + if (!IS_ALIGNED(reg, map->reg_stride)) return -EINVAL; map->lock(map->lock_arg); @@ -1589,9 +2856,46 @@ int regmap_read(struct regmap *map, unsigned int reg, unsigned int *val) EXPORT_SYMBOL_GPL(regmap_read); /** - * regmap_raw_read(): Read raw data from the device + * regmap_read_bypassed() - Read a value from a single register direct + * from the device, bypassing the cache * - * @map: Register map to write to + * @map: Register map to read from + * @reg: Register to be read from + * @val: Pointer to store read value + * + * A value of zero will be returned on success, a negative errno will + * be returned in error cases. + */ +int regmap_read_bypassed(struct regmap *map, unsigned int reg, unsigned int *val) +{ + int ret; + bool bypass, cache_only; + + if (!IS_ALIGNED(reg, map->reg_stride)) + return -EINVAL; + + map->lock(map->lock_arg); + + bypass = map->cache_bypass; + cache_only = map->cache_only; + map->cache_bypass = true; + map->cache_only = false; + + ret = _regmap_read(map, reg, val); + + map->cache_bypass = bypass; + map->cache_only = cache_only; + + map->unlock(map->lock_arg); + + return ret; +} +EXPORT_SYMBOL_GPL(regmap_read_bypassed); + +/** + * regmap_raw_read() - Read raw data from the device + * + * @map: Register map to read from * @reg: First register to be read from * @val: Pointer to store read value * @val_len: Size of data to read @@ -1607,26 +2911,61 @@ int regmap_raw_read(struct regmap *map, unsigned int reg, void *val, unsigned int v; int ret, i; - if (!map->bus) - return -EINVAL; if (val_len % map->format.val_bytes) return -EINVAL; - if (reg % map->reg_stride) + if (!IS_ALIGNED(reg, map->reg_stride)) + return -EINVAL; + if (val_count == 0) return -EINVAL; map->lock(map->lock_arg); if (regmap_volatile_range(map, reg, val_count) || map->cache_bypass || map->cache_type == REGCACHE_NONE) { - /* Physical block read if there's no cache involved */ - ret = _regmap_raw_read(map, reg, val, val_len); + size_t chunk_count, chunk_bytes; + size_t chunk_regs = val_count; + + if (!map->cache_bypass && map->cache_only) { + ret = -EBUSY; + goto out; + } + + if (!map->read) { + ret = -ENOTSUPP; + goto out; + } + + if (map->use_single_read) + chunk_regs = 1; + else if (map->max_raw_read && val_len > map->max_raw_read) + chunk_regs = map->max_raw_read / val_bytes; + + chunk_count = val_count / chunk_regs; + chunk_bytes = chunk_regs * val_bytes; + + /* Read bytes that fit into whole chunks */ + for (i = 0; i < chunk_count; i++) { + ret = _regmap_raw_read(map, reg, val, chunk_bytes, false); + if (ret != 0) + goto out; + + reg += regmap_get_offset(map, chunk_regs); + val += chunk_bytes; + val_len -= chunk_bytes; + } + /* Read remaining bytes */ + if (val_len) { + ret = _regmap_raw_read(map, reg, val, val_len, false); + if (ret != 0) + goto out; + } } else { /* Otherwise go word by word for the cache; should be low * cost as we expect to hit the cache. */ for (i = 0; i < val_count; i++) { - ret = _regmap_read(map, reg + (i * map->reg_stride), + ret = _regmap_read(map, reg + regmap_get_offset(map, i), &v); if (ret != 0) goto out; @@ -1643,6 +2982,85 @@ int regmap_raw_read(struct regmap *map, unsigned int reg, void *val, EXPORT_SYMBOL_GPL(regmap_raw_read); /** + * regmap_noinc_read(): Read data from a register without incrementing the + * register number + * + * @map: Register map to read from + * @reg: Register to read from + * @val: Pointer to data buffer + * @val_len: Length of output buffer in bytes. + * + * The regmap API usually assumes that bulk read operations will read a + * range of registers. Some devices have certain registers for which a read + * operation read will read from an internal FIFO. + * + * The target register must be volatile but registers after it can be + * completely unrelated cacheable registers. + * + * This will attempt multiple reads as required to read val_len bytes. + * + * A value of zero will be returned on success, a negative errno will be + * returned in error cases. + */ +int regmap_noinc_read(struct regmap *map, unsigned int reg, + void *val, size_t val_len) +{ + size_t read_len; + int ret; + + if (!map->read) + return -ENOTSUPP; + + if (val_len % map->format.val_bytes) + return -EINVAL; + if (!IS_ALIGNED(reg, map->reg_stride)) + return -EINVAL; + if (val_len == 0) + return -EINVAL; + + map->lock(map->lock_arg); + + if (!regmap_volatile(map, reg) || !regmap_readable_noinc(map, reg)) { + ret = -EINVAL; + goto out_unlock; + } + + /* + * We have not defined the FIFO semantics for cache, as the + * cache is just one value deep. Should we return the last + * written value? Just avoid this by always reading the FIFO + * even when using cache. Cache only will not work. + */ + if (!map->cache_bypass && map->cache_only) { + ret = -EBUSY; + goto out_unlock; + } + + /* Use the accelerated operation if we can */ + if (map->bus->reg_noinc_read) { + ret = regmap_noinc_readwrite(map, reg, val, val_len, false); + goto out_unlock; + } + + while (val_len) { + if (map->max_raw_read && map->max_raw_read < val_len) + read_len = map->max_raw_read; + else + read_len = val_len; + ret = _regmap_raw_read(map, reg, val, read_len, true); + if (ret) + goto out_unlock; + val = ((u8 *)val) + read_len; + val_len -= read_len; + } + +out_unlock: + map->unlock(map->lock_arg); + return ret; +} +EXPORT_SYMBOL_GPL(regmap_noinc_read); + +/** * regmap_field_read(): Read a value to a single register field * * @field: Register field to read from @@ -1668,9 +3086,87 @@ int regmap_field_read(struct regmap_field *field, unsigned int *val) EXPORT_SYMBOL_GPL(regmap_field_read); /** - * regmap_bulk_read(): Read multiple registers from the device + * regmap_fields_read() - Read a value to a single register field with port ID * - * @map: Register map to write to + * @field: Register field to read from + * @id: port ID + * @val: Pointer to store read value + * + * A value of zero will be returned on success, a negative errno will + * be returned in error cases. + */ +int regmap_fields_read(struct regmap_field *field, unsigned int id, + unsigned int *val) +{ + int ret; + unsigned int reg_val; + + if (id >= field->id_size) + return -EINVAL; + + ret = regmap_read(field->regmap, + field->reg + (field->id_offset * id), + ®_val); + if (ret != 0) + return ret; + + reg_val &= field->mask; + reg_val >>= field->shift; + *val = reg_val; + + return ret; +} +EXPORT_SYMBOL_GPL(regmap_fields_read); + +static int _regmap_bulk_read(struct regmap *map, unsigned int reg, + const unsigned int *regs, void *val, size_t val_count) +{ + u32 *u32 = val; + u16 *u16 = val; + u8 *u8 = val; + int ret, i; + + map->lock(map->lock_arg); + + for (i = 0; i < val_count; i++) { + unsigned int ival; + + if (regs) { + if (!IS_ALIGNED(regs[i], map->reg_stride)) { + ret = -EINVAL; + goto out; + } + ret = _regmap_read(map, regs[i], &ival); + } else { + ret = _regmap_read(map, reg + regmap_get_offset(map, i), &ival); + } + if (ret != 0) + goto out; + + switch (map->format.val_bytes) { + case 4: + u32[i] = ival; + break; + case 2: + u16[i] = ival; + break; + case 1: + u8[i] = ival; + break; + default: + ret = -EINVAL; + goto out; + } + } +out: + map->unlock(map->lock_arg); + return ret; +} + +/** + * regmap_bulk_read() - Read multiple sequential registers from the device + * + * @map: Register map to read from * @reg: First register to be read from * @val: Pointer to store read value, in native register size for device * @val_count: Number of registers to read @@ -1685,134 +3181,156 @@ int regmap_bulk_read(struct regmap *map, unsigned int reg, void *val, size_t val_bytes = map->format.val_bytes; bool vol = regmap_volatile_range(map, reg, val_count); - if (!map->bus) - return -EINVAL; - if (!map->format.parse_inplace) + if (!IS_ALIGNED(reg, map->reg_stride)) return -EINVAL; - if (reg % map->reg_stride) + if (val_count == 0) return -EINVAL; - if (vol || map->cache_type == REGCACHE_NONE) { - /* - * Some devices does not support bulk read, for - * them we have a series of single read operations. - */ - if (map->use_single_rw) { - for (i = 0; i < val_count; i++) { - ret = regmap_raw_read(map, - reg + (i * map->reg_stride), - val + (i * val_bytes), - val_bytes); - if (ret != 0) - return ret; - } - } else { - ret = regmap_raw_read(map, reg, val, - val_bytes * val_count); - if (ret != 0) - return ret; - } + if (map->read && map->format.parse_inplace && (vol || map->cache_type == REGCACHE_NONE)) { + ret = regmap_raw_read(map, reg, val, val_bytes * val_count); + if (ret != 0) + return ret; for (i = 0; i < val_count * val_bytes; i += val_bytes) map->format.parse_inplace(val + i); } else { - for (i = 0; i < val_count; i++) { - unsigned int ival; - ret = regmap_read(map, reg + (i * map->reg_stride), - &ival); - if (ret != 0) - return ret; - memcpy(val + (i * val_bytes), &ival, val_bytes); - } + ret = _regmap_bulk_read(map, reg, NULL, val, val_count); } - - return 0; + if (!ret) + trace_regmap_bulk_read(map, reg, val, val_bytes * val_count); + return ret; } EXPORT_SYMBOL_GPL(regmap_bulk_read); +/** + * regmap_multi_reg_read() - Read multiple non-sequential registers from the device + * + * @map: Register map to read from + * @regs: Array of registers to read from + * @val: Pointer to store read value, in native register size for device + * @val_count: Number of registers to read + * + * A value of zero will be returned on success, a negative errno will + * be returned in error cases. + */ +int regmap_multi_reg_read(struct regmap *map, const unsigned int *regs, void *val, + size_t val_count) +{ + if (val_count == 0) + return -EINVAL; + + return _regmap_bulk_read(map, 0, regs, val, val_count); +} +EXPORT_SYMBOL_GPL(regmap_multi_reg_read); + static int _regmap_update_bits(struct regmap *map, unsigned int reg, unsigned int mask, unsigned int val, - bool *change) + bool *change, bool force_write) { int ret; unsigned int tmp, orig; - ret = _regmap_read(map, reg, &orig); - if (ret != 0) - return ret; - - tmp = orig & ~mask; - tmp |= val & mask; + if (change) + *change = false; - if (tmp != orig) { - ret = _regmap_write(map, reg, tmp); - *change = true; + if (regmap_volatile(map, reg) && map->reg_update_bits) { + reg = regmap_reg_addr(map, reg); + ret = map->reg_update_bits(map->bus_context, reg, mask, val); + if (ret == 0 && change) + *change = true; } else { - *change = false; + ret = _regmap_read(map, reg, &orig); + if (ret != 0) + return ret; + + tmp = orig & ~mask; + tmp |= val & mask; + + if (force_write || (tmp != orig) || map->force_write_field) { + ret = _regmap_write(map, reg, tmp); + if (ret == 0 && change) + *change = true; + } } return ret; } /** - * regmap_update_bits: Perform a read/modify/write cycle on the register map + * regmap_update_bits_base() - Perform a read/modify/write cycle on a register * * @map: Register map to update * @reg: Register to update * @mask: Bitmask to change * @val: New value for bitmask + * @change: Boolean indicating if a write was done + * @async: Boolean indicating asynchronously + * @force: Boolean indicating use force update + * + * Perform a read/modify/write cycle on a register map with change, async, force + * options. + * + * If async is true: + * + * With most buses the read must be done synchronously so this is most useful + * for devices with a cache which do not need to interact with the hardware to + * determine the current register value. * * Returns zero for success, a negative number on error. */ -int regmap_update_bits(struct regmap *map, unsigned int reg, - unsigned int mask, unsigned int val) +int regmap_update_bits_base(struct regmap *map, unsigned int reg, + unsigned int mask, unsigned int val, + bool *change, bool async, bool force) { - bool change; int ret; map->lock(map->lock_arg); - ret = _regmap_update_bits(map, reg, mask, val, &change); + + map->async = async; + + ret = _regmap_update_bits(map, reg, mask, val, change, force); + + map->async = false; + map->unlock(map->lock_arg); return ret; } -EXPORT_SYMBOL_GPL(regmap_update_bits); +EXPORT_SYMBOL_GPL(regmap_update_bits_base); /** - * regmap_update_bits_check: Perform a read/modify/write cycle on the - * register map and report if updated + * regmap_test_bits() - Check if all specified bits are set in a register. * - * @map: Register map to update - * @reg: Register to update - * @mask: Bitmask to change - * @val: New value for bitmask - * @change: Boolean indicating if a write was done + * @map: Register map to operate on + * @reg: Register to read from + * @bits: Bits to test * - * Returns zero for success, a negative number on error. + * Returns 0 if at least one of the tested bits is not set, 1 if all tested + * bits are set and a negative error number if the underlying regmap_read() + * fails. */ -int regmap_update_bits_check(struct regmap *map, unsigned int reg, - unsigned int mask, unsigned int val, - bool *change) +int regmap_test_bits(struct regmap *map, unsigned int reg, unsigned int bits) { + unsigned int val; int ret; - map->lock(map->lock_arg); - ret = _regmap_update_bits(map, reg, mask, val, change); - map->unlock(map->lock_arg); - return ret; + ret = regmap_read(map, reg, &val); + if (ret) + return ret; + + return (val & bits) == bits; } -EXPORT_SYMBOL_GPL(regmap_update_bits_check); +EXPORT_SYMBOL_GPL(regmap_test_bits); void regmap_async_complete_cb(struct regmap_async *async, int ret) { struct regmap *map = async->map; bool wake; - trace_regmap_async_io_complete(map->dev); + trace_regmap_async_io_complete(map); spin_lock(&map->async_lock); - - list_del(&async->list); + list_move(&async->list, &map->async_free); wake = list_empty(&map->async_list); if (ret != 0) @@ -1820,8 +3338,6 @@ void regmap_async_complete_cb(struct regmap_async *async, int ret) spin_unlock(&map->async_lock); - schedule_work(&async->cleanup); - if (wake) wake_up(&map->async_waitq); } @@ -1840,7 +3356,7 @@ static int regmap_async_is_done(struct regmap *map) } /** - * regmap_async_complete: Ensure all asynchronous I/O has completed. + * regmap_async_complete - Ensure all asynchronous I/O has completed. * * @map: Map to operate on. * @@ -1853,10 +3369,10 @@ int regmap_async_complete(struct regmap *map) int ret; /* Nothing to do with no async support */ - if (!map->bus->async_write) + if (!map->bus || !map->bus->async_write) return 0; - trace_regmap_async_complete_start(map->dev); + trace_regmap_async_complete_start(map); wait_event(map->async_waitq, regmap_async_is_done(map)); @@ -1865,15 +3381,15 @@ int regmap_async_complete(struct regmap *map) map->async_ret = 0; spin_unlock_irqrestore(&map->async_lock, flags); - trace_regmap_async_complete_done(map->dev); + trace_regmap_async_complete_done(map); return ret; } EXPORT_SYMBOL_GPL(regmap_async_complete); /** - * regmap_register_patch: Register and apply register updates to be applied - * on device initialistion + * regmap_register_patch - Register and apply register updates to be applied + * on device initialistion * * @map: Register map to apply updates to. * @regs: Values to update. @@ -1884,53 +3400,56 @@ EXPORT_SYMBOL_GPL(regmap_async_complete); * apply them immediately. Typically this is used to apply * corrections to be applied to the device defaults on startup, such * as the updates some vendors provide to undocumented registers. + * + * The caller must ensure that this function cannot be called + * concurrently with either itself or regcache_sync(). */ -int regmap_register_patch(struct regmap *map, const struct reg_default *regs, +int regmap_register_patch(struct regmap *map, const struct reg_sequence *regs, int num_regs) { - int i, ret; + struct reg_sequence *p; + int ret; bool bypass; - /* If needed the implementation can be extended to support this */ - if (map->patch) - return -EBUSY; + if (WARN_ONCE(num_regs <= 0, "invalid registers number (%d)\n", + num_regs)) + return 0; + + p = krealloc(map->patch, + sizeof(struct reg_sequence) * (map->patch_regs + num_regs), + GFP_KERNEL); + if (p) { + memcpy(p + map->patch_regs, regs, num_regs * sizeof(*regs)); + map->patch = p; + map->patch_regs += num_regs; + } else { + return -ENOMEM; + } map->lock(map->lock_arg); bypass = map->cache_bypass; map->cache_bypass = true; + map->async = true; - /* Write out first; it's useful to apply even if we fail later. */ - for (i = 0; i < num_regs; i++) { - ret = _regmap_write(map, regs[i].reg, regs[i].def); - if (ret != 0) { - dev_err(map->dev, "Failed to write %x = %x: %d\n", - regs[i].reg, regs[i].def, ret); - goto out; - } - } + ret = _regmap_multi_reg_write(map, regs, num_regs); - map->patch = kcalloc(num_regs, sizeof(struct reg_default), GFP_KERNEL); - if (map->patch != NULL) { - memcpy(map->patch, regs, - num_regs * sizeof(struct reg_default)); - map->patch_regs = num_regs; - } else { - ret = -ENOMEM; - } - -out: + map->async = false; map->cache_bypass = bypass; map->unlock(map->lock_arg); + regmap_async_complete(map); + return ret; } EXPORT_SYMBOL_GPL(regmap_register_patch); -/* - * regmap_get_val_bytes(): Report the size of a register value +/** + * regmap_get_val_bytes() - Report the size of a register value + * + * @map: Register map to operate on. * * Report the size of a register value, mainly intended to for use by * generic infrastructure built on top of regmap. @@ -1944,6 +3463,59 @@ int regmap_get_val_bytes(struct regmap *map) } EXPORT_SYMBOL_GPL(regmap_get_val_bytes); +/** + * regmap_get_max_register() - Report the max register value + * + * @map: Register map to operate on. + * + * Report the max register value, mainly intended to for use by + * generic infrastructure built on top of regmap. + */ +int regmap_get_max_register(struct regmap *map) +{ + return map->max_register_is_set ? map->max_register : -EINVAL; +} +EXPORT_SYMBOL_GPL(regmap_get_max_register); + +/** + * regmap_get_reg_stride() - Report the register address stride + * + * @map: Register map to operate on. + * + * Report the register address stride, mainly intended to for use by + * generic infrastructure built on top of regmap. + */ +int regmap_get_reg_stride(struct regmap *map) +{ + return map->reg_stride; +} +EXPORT_SYMBOL_GPL(regmap_get_reg_stride); + +/** + * regmap_might_sleep() - Returns whether a regmap access might sleep. + * + * @map: Register map to operate on. + * + * Returns true if an access to the register might sleep, else false. + */ +bool regmap_might_sleep(struct regmap *map) +{ + return map->can_sleep; +} +EXPORT_SYMBOL_GPL(regmap_might_sleep); + +int regmap_parse_val(struct regmap *map, const void *buf, + unsigned int *val) +{ + if (!map->format.parse_val) + return -EINVAL; + + *val = map->format.parse_val(buf); + + return 0; +} +EXPORT_SYMBOL_GPL(regmap_parse_val); + static int __init regmap_initcall(void) { regmap_debugfs_initcall(); diff --git a/drivers/base/regmap/trace.h b/drivers/base/regmap/trace.h new file mode 100644 index 000000000000..bcc5a8b226a6 --- /dev/null +++ b/drivers/base/regmap/trace.h @@ -0,0 +1,284 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM regmap + +#if !defined(_TRACE_REGMAP_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_REGMAP_H + +#include <linux/ktime.h> +#include <linux/tracepoint.h> + +#include "internal.h" + +/* + * Log register events + */ +DECLARE_EVENT_CLASS(regmap_reg, + + TP_PROTO(struct regmap *map, unsigned int reg, + unsigned int val), + + TP_ARGS(map, reg, val), + + TP_STRUCT__entry( + __string( name, regmap_name(map) ) + __field( unsigned int, reg ) + __field( unsigned int, val ) + ), + + TP_fast_assign( + __assign_str(name); + __entry->reg = reg; + __entry->val = val; + ), + + TP_printk("%s reg=%x val=%x", __get_str(name), __entry->reg, __entry->val) +); + +DEFINE_EVENT(regmap_reg, regmap_reg_write, + + TP_PROTO(struct regmap *map, unsigned int reg, + unsigned int val), + + TP_ARGS(map, reg, val) +); + +DEFINE_EVENT(regmap_reg, regmap_reg_read, + + TP_PROTO(struct regmap *map, unsigned int reg, + unsigned int val), + + TP_ARGS(map, reg, val) +); + +DEFINE_EVENT(regmap_reg, regmap_reg_read_cache, + + TP_PROTO(struct regmap *map, unsigned int reg, + unsigned int val), + + TP_ARGS(map, reg, val) +); + +DECLARE_EVENT_CLASS(regmap_bulk, + + TP_PROTO(struct regmap *map, unsigned int reg, + const void *val, int val_len), + + TP_ARGS(map, reg, val, val_len), + + TP_STRUCT__entry( + __string(name, regmap_name(map)) + __field(unsigned int, reg) + __dynamic_array(char, buf, val_len) + __field(int, val_len) + ), + + TP_fast_assign( + __assign_str(name); + __entry->reg = reg; + __entry->val_len = val_len; + memcpy(__get_dynamic_array(buf), val, val_len); + ), + + TP_printk("%s reg=%x val=%s", __get_str(name), __entry->reg, + __print_hex(__get_dynamic_array(buf), __entry->val_len)) +); + +DEFINE_EVENT(regmap_bulk, regmap_bulk_write, + + TP_PROTO(struct regmap *map, unsigned int reg, + const void *val, int val_len), + + TP_ARGS(map, reg, val, val_len) +); + +DEFINE_EVENT(regmap_bulk, regmap_bulk_read, + + TP_PROTO(struct regmap *map, unsigned int reg, + const void *val, int val_len), + + TP_ARGS(map, reg, val, val_len) +); + +DECLARE_EVENT_CLASS(regmap_block, + + TP_PROTO(struct regmap *map, unsigned int reg, int count), + + TP_ARGS(map, reg, count), + + TP_STRUCT__entry( + __string( name, regmap_name(map) ) + __field( unsigned int, reg ) + __field( int, count ) + ), + + TP_fast_assign( + __assign_str(name); + __entry->reg = reg; + __entry->count = count; + ), + + TP_printk("%s reg=%x count=%d", __get_str(name), __entry->reg, __entry->count) +); + +DEFINE_EVENT(regmap_block, regmap_hw_read_start, + + TP_PROTO(struct regmap *map, unsigned int reg, int count), + + TP_ARGS(map, reg, count) +); + +DEFINE_EVENT(regmap_block, regmap_hw_read_done, + + TP_PROTO(struct regmap *map, unsigned int reg, int count), + + TP_ARGS(map, reg, count) +); + +DEFINE_EVENT(regmap_block, regmap_hw_write_start, + + TP_PROTO(struct regmap *map, unsigned int reg, int count), + + TP_ARGS(map, reg, count) +); + +DEFINE_EVENT(regmap_block, regmap_hw_write_done, + + TP_PROTO(struct regmap *map, unsigned int reg, int count), + + TP_ARGS(map, reg, count) +); + +TRACE_EVENT(regcache_sync, + + TP_PROTO(struct regmap *map, const char *type, + const char *status), + + TP_ARGS(map, type, status), + + TP_STRUCT__entry( + __string( name, regmap_name(map) ) + __string( status, status ) + __string( type, type ) + ), + + TP_fast_assign( + __assign_str(name); + __assign_str(status); + __assign_str(type); + ), + + TP_printk("%s type=%s status=%s", __get_str(name), + __get_str(type), __get_str(status)) +); + +DECLARE_EVENT_CLASS(regmap_bool, + + TP_PROTO(struct regmap *map, bool flag), + + TP_ARGS(map, flag), + + TP_STRUCT__entry( + __string( name, regmap_name(map) ) + __field( int, flag ) + ), + + TP_fast_assign( + __assign_str(name); + __entry->flag = flag; + ), + + TP_printk("%s flag=%d", __get_str(name), __entry->flag) +); + +DEFINE_EVENT(regmap_bool, regmap_cache_only, + + TP_PROTO(struct regmap *map, bool flag), + + TP_ARGS(map, flag) +); + +DEFINE_EVENT(regmap_bool, regmap_cache_bypass, + + TP_PROTO(struct regmap *map, bool flag), + + TP_ARGS(map, flag) +); + +DECLARE_EVENT_CLASS(regmap_async, + + TP_PROTO(struct regmap *map), + + TP_ARGS(map), + + TP_STRUCT__entry( + __string( name, regmap_name(map) ) + ), + + TP_fast_assign( + __assign_str(name); + ), + + TP_printk("%s", __get_str(name)) +); + +DEFINE_EVENT(regmap_block, regmap_async_write_start, + + TP_PROTO(struct regmap *map, unsigned int reg, int count), + + TP_ARGS(map, reg, count) +); + +DEFINE_EVENT(regmap_async, regmap_async_io_complete, + + TP_PROTO(struct regmap *map), + + TP_ARGS(map) +); + +DEFINE_EVENT(regmap_async, regmap_async_complete_start, + + TP_PROTO(struct regmap *map), + + TP_ARGS(map) +); + +DEFINE_EVENT(regmap_async, regmap_async_complete_done, + + TP_PROTO(struct regmap *map), + + TP_ARGS(map) +); + +TRACE_EVENT(regcache_drop_region, + + TP_PROTO(struct regmap *map, unsigned int from, + unsigned int to), + + TP_ARGS(map, from, to), + + TP_STRUCT__entry( + __string( name, regmap_name(map) ) + __field( unsigned int, from ) + __field( unsigned int, to ) + ), + + TP_fast_assign( + __assign_str(name); + __entry->from = from; + __entry->to = to; + ), + + TP_printk("%s %u-%u", __get_str(name), __entry->from, __entry->to) +); + +#endif /* _TRACE_REGMAP_H */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . + +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE trace + +/* This part must be outside protection */ +#include <trace/define_trace.h> diff --git a/drivers/base/reservation.c b/drivers/base/reservation.c deleted file mode 100644 index a73fbf3b8e56..000000000000 --- a/drivers/base/reservation.c +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (C) 2012-2013 Canonical Ltd - * - * Based on bo.c which bears the following copyright notice, - * but is dual licensed: - * - * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA - * All Rights Reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sub license, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice (including the - * next paragraph) shall be included in all copies or substantial portions - * of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, - * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR - * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE - * USE OR OTHER DEALINGS IN THE SOFTWARE. - * - **************************************************************************/ -/* - * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> - */ - -#include <linux/reservation.h> -#include <linux/export.h> - -DEFINE_WW_CLASS(reservation_ww_class); -EXPORT_SYMBOL(reservation_ww_class); diff --git a/drivers/base/soc.c b/drivers/base/soc.c index 72b5e7280d14..282c38aece0d 100644 --- a/drivers/base/soc.c +++ b/drivers/base/soc.c @@ -1,26 +1,26 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Copyright (C) ST-Ericsson SA 2011 * * Author: Lee Jones <lee.jones@linaro.org> for ST-Ericsson. - * License terms: GNU General Public License (GPL), version 2 */ #include <linux/sysfs.h> -#include <linux/module.h> #include <linux/init.h> +#include <linux/of.h> #include <linux/stat.h> #include <linux/slab.h> #include <linux/idr.h> #include <linux/spinlock.h> #include <linux/sys_soc.h> #include <linux/err.h> +#include <linux/glob.h> static DEFINE_IDA(soc_ida); -static DEFINE_SPINLOCK(soc_lock); -static ssize_t soc_info_get(struct device *dev, - struct device_attribute *attr, - char *buf); +/* Prototype to allow declarations of DEVICE_ATTR(<foo>) before soc_info_show */ +static ssize_t soc_info_show(struct device *dev, struct device_attribute *attr, + char *buf); struct soc_device { struct device dev; @@ -28,14 +28,16 @@ struct soc_device { int soc_dev_num; }; -static struct bus_type soc_bus_type = { +static const struct bus_type soc_bus_type = { .name = "soc", }; +static bool soc_bus_registered; -static DEVICE_ATTR(machine, S_IRUGO, soc_info_get, NULL); -static DEVICE_ATTR(family, S_IRUGO, soc_info_get, NULL); -static DEVICE_ATTR(soc_id, S_IRUGO, soc_info_get, NULL); -static DEVICE_ATTR(revision, S_IRUGO, soc_info_get, NULL); +static DEVICE_ATTR(machine, 0444, soc_info_show, NULL); +static DEVICE_ATTR(family, 0444, soc_info_show, NULL); +static DEVICE_ATTR(serial_number, 0444, soc_info_show, NULL); +static DEVICE_ATTR(soc_id, 0444, soc_info_show, NULL); +static DEVICE_ATTR(revision, 0444, soc_info_show, NULL); struct device *soc_device_to_device(struct soc_device *soc_dev) { @@ -43,51 +45,53 @@ struct device *soc_device_to_device(struct soc_device *soc_dev) } static umode_t soc_attribute_mode(struct kobject *kobj, - struct attribute *attr, - int index) + struct attribute *attr, + int index) { - struct device *dev = container_of(kobj, struct device, kobj); + struct device *dev = kobj_to_dev(kobj); struct soc_device *soc_dev = container_of(dev, struct soc_device, dev); - if ((attr == &dev_attr_machine.attr) - && (soc_dev->attr->machine != NULL)) + if ((attr == &dev_attr_machine.attr) && soc_dev->attr->machine) return attr->mode; - if ((attr == &dev_attr_family.attr) - && (soc_dev->attr->family != NULL)) + if ((attr == &dev_attr_family.attr) && soc_dev->attr->family) return attr->mode; - if ((attr == &dev_attr_revision.attr) - && (soc_dev->attr->revision != NULL)) + if ((attr == &dev_attr_revision.attr) && soc_dev->attr->revision) + return attr->mode; + if ((attr == &dev_attr_serial_number.attr) && soc_dev->attr->serial_number) + return attr->mode; + if ((attr == &dev_attr_soc_id.attr) && soc_dev->attr->soc_id) return attr->mode; - if ((attr == &dev_attr_soc_id.attr) - && (soc_dev->attr->soc_id != NULL)) - return attr->mode; - /* Unknown or unfilled attribute. */ + /* Unknown or unfilled attribute */ return 0; } -static ssize_t soc_info_get(struct device *dev, - struct device_attribute *attr, - char *buf) +static ssize_t soc_info_show(struct device *dev, struct device_attribute *attr, + char *buf) { struct soc_device *soc_dev = container_of(dev, struct soc_device, dev); + const char *output; if (attr == &dev_attr_machine) - return sprintf(buf, "%s\n", soc_dev->attr->machine); - if (attr == &dev_attr_family) - return sprintf(buf, "%s\n", soc_dev->attr->family); - if (attr == &dev_attr_revision) - return sprintf(buf, "%s\n", soc_dev->attr->revision); - if (attr == &dev_attr_soc_id) - return sprintf(buf, "%s\n", soc_dev->attr->soc_id); - - return -EINVAL; - + output = soc_dev->attr->machine; + else if (attr == &dev_attr_family) + output = soc_dev->attr->family; + else if (attr == &dev_attr_revision) + output = soc_dev->attr->revision; + else if (attr == &dev_attr_serial_number) + output = soc_dev->attr->serial_number; + else if (attr == &dev_attr_soc_id) + output = soc_dev->attr->soc_id; + else + return -EINVAL; + + return sysfs_emit(buf, "%s\n", output); } static struct attribute *soc_attr[] = { &dev_attr_machine.attr, &dev_attr_family.attr, + &dev_attr_serial_number.attr, &dev_attr_soc_id.attr, &dev_attr_revision.attr, NULL, @@ -98,44 +102,63 @@ static const struct attribute_group soc_attr_group = { .is_visible = soc_attribute_mode, }; -static const struct attribute_group *soc_attr_groups[] = { - &soc_attr_group, - NULL, -}; - static void soc_release(struct device *dev) { struct soc_device *soc_dev = container_of(dev, struct soc_device, dev); + ida_free(&soc_ida, soc_dev->soc_dev_num); + kfree(soc_dev->dev.groups); kfree(soc_dev); } +static void soc_device_get_machine(struct soc_device_attribute *soc_dev_attr) +{ + struct device_node *np; + + if (soc_dev_attr->machine) + return; + + np = of_find_node_by_path("/"); + of_property_read_string(np, "model", &soc_dev_attr->machine); + of_node_put(np); +} + +static struct soc_device_attribute *early_soc_dev_attr; + struct soc_device *soc_device_register(struct soc_device_attribute *soc_dev_attr) { struct soc_device *soc_dev; + const struct attribute_group **soc_attr_groups; int ret; + soc_device_get_machine(soc_dev_attr); + + if (!soc_bus_registered) { + if (early_soc_dev_attr) + return ERR_PTR(-EBUSY); + early_soc_dev_attr = soc_dev_attr; + return NULL; + } + soc_dev = kzalloc(sizeof(*soc_dev), GFP_KERNEL); if (!soc_dev) { - ret = -ENOMEM; + ret = -ENOMEM; goto out1; } - /* Fetch a unique (reclaimable) SOC ID. */ - do { - if (!ida_pre_get(&soc_ida, GFP_KERNEL)) { - ret = -ENOMEM; - goto out2; - } - - spin_lock(&soc_lock); - ret = ida_get_new(&soc_ida, &soc_dev->soc_dev_num); - spin_unlock(&soc_lock); - - } while (ret == -EAGAIN); + soc_attr_groups = kcalloc(3, sizeof(*soc_attr_groups), GFP_KERNEL); + if (!soc_attr_groups) { + ret = -ENOMEM; + goto out2; + } + soc_attr_groups[0] = &soc_attr_group; + soc_attr_groups[1] = soc_dev_attr->custom_attr_group; - if (ret) - goto out2; + /* Fetch a unique (reclaimable) SOC ID. */ + ret = ida_alloc(&soc_ida, GFP_KERNEL); + if (ret < 0) + goto out3; + soc_dev->soc_dev_num = ret; soc_dev->attr = soc_dev_attr; soc_dev->dev.bus = &soc_bus_type; @@ -145,37 +168,115 @@ struct soc_device *soc_device_register(struct soc_device_attribute *soc_dev_attr dev_set_name(&soc_dev->dev, "soc%d", soc_dev->soc_dev_num); ret = device_register(&soc_dev->dev); - if (ret) - goto out3; + if (ret) { + put_device(&soc_dev->dev); + return ERR_PTR(ret); + } return soc_dev; out3: - ida_remove(&soc_ida, soc_dev->soc_dev_num); + kfree(soc_attr_groups); out2: kfree(soc_dev); out1: return ERR_PTR(ret); } +EXPORT_SYMBOL_GPL(soc_device_register); -/* Ensure soc_dev->attr is freed prior to calling soc_device_unregister. */ +/* Ensure soc_dev->attr is freed after calling soc_device_unregister. */ void soc_device_unregister(struct soc_device *soc_dev) { - ida_remove(&soc_ida, soc_dev->soc_dev_num); - device_unregister(&soc_dev->dev); + early_soc_dev_attr = NULL; } +EXPORT_SYMBOL_GPL(soc_device_unregister); static int __init soc_bus_register(void) { - return bus_register(&soc_bus_type); + int ret; + + ret = bus_register(&soc_bus_type); + if (ret) + return ret; + soc_bus_registered = true; + + if (early_soc_dev_attr) + return PTR_ERR(soc_device_register(early_soc_dev_attr)); + + return 0; } core_initcall(soc_bus_register); -static void __exit soc_bus_unregister(void) +static int soc_device_match_attr(const struct soc_device_attribute *attr, + const struct soc_device_attribute *match) +{ + if (match->machine && + (!attr->machine || !glob_match(match->machine, attr->machine))) + return 0; + + if (match->family && + (!attr->family || !glob_match(match->family, attr->family))) + return 0; + + if (match->revision && + (!attr->revision || !glob_match(match->revision, attr->revision))) + return 0; + + if (match->soc_id && + (!attr->soc_id || !glob_match(match->soc_id, attr->soc_id))) + return 0; + + return 1; +} + +static int soc_device_match_one(struct device *dev, void *arg) { - ida_destroy(&soc_ida); + struct soc_device *soc_dev = container_of(dev, struct soc_device, dev); + + return soc_device_match_attr(soc_dev->attr, arg); +} - bus_unregister(&soc_bus_type); +/* + * soc_device_match - identify the SoC in the machine + * @matches: zero-terminated array of possible matches + * + * returns the first matching entry of the argument array, or NULL + * if none of them match. + * + * This function is meant as a helper in place of of_match_node() + * in cases where either no device tree is available or the information + * in a device node is insufficient to identify a particular variant + * by its compatible strings or other properties. For new devices, + * the DT binding should always provide unique compatible strings + * that allow the use of of_match_node() instead. + * + * The calling function can use the .data entry of the + * soc_device_attribute to pass a structure or function pointer for + * each entry. + */ +const struct soc_device_attribute *soc_device_match( + const struct soc_device_attribute *matches) +{ + int ret; + + if (!matches) + return NULL; + + while (matches->machine || matches->family || matches->revision || + matches->soc_id) { + ret = bus_for_each_dev(&soc_bus_type, NULL, (void *)matches, + soc_device_match_one); + if (ret < 0 && early_soc_dev_attr) + ret = soc_device_match_attr(early_soc_dev_attr, + matches); + if (ret < 0) + return NULL; + if (ret) + return matches; + + matches++; + } + return NULL; } -module_exit(soc_bus_unregister); +EXPORT_SYMBOL_GPL(soc_device_match); diff --git a/drivers/base/swnode.c b/drivers/base/swnode.c new file mode 100644 index 000000000000..16a8301c25d6 --- /dev/null +++ b/drivers/base/swnode.c @@ -0,0 +1,1144 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Software nodes for the firmware node framework. + * + * Copyright (C) 2018, Intel Corporation + * Author: Heikki Krogerus <heikki.krogerus@linux.intel.com> + */ + +#include <linux/container_of.h> +#include <linux/device.h> +#include <linux/err.h> +#include <linux/export.h> +#include <linux/idr.h> +#include <linux/init.h> +#include <linux/kobject.h> +#include <linux/kstrtox.h> +#include <linux/list.h> +#include <linux/property.h> +#include <linux/slab.h> +#include <linux/spinlock.h> +#include <linux/string.h> +#include <linux/sysfs.h> +#include <linux/types.h> + +#include "base.h" + +struct swnode { + struct kobject kobj; + struct fwnode_handle fwnode; + const struct software_node *node; + int id; + + /* hierarchy */ + struct ida child_ids; + struct list_head entry; + struct list_head children; + struct swnode *parent; + + unsigned int allocated:1; + unsigned int managed:1; +}; + +static DEFINE_IDA(swnode_root_ids); +static struct kset *swnode_kset; + +#define kobj_to_swnode(_kobj_) container_of(_kobj_, struct swnode, kobj) + +static const struct fwnode_operations software_node_ops; + +bool is_software_node(const struct fwnode_handle *fwnode) +{ + return !IS_ERR_OR_NULL(fwnode) && fwnode->ops == &software_node_ops; +} +EXPORT_SYMBOL_GPL(is_software_node); + +#define to_swnode(__fwnode) \ + ({ \ + typeof(__fwnode) __to_swnode_fwnode = __fwnode; \ + \ + is_software_node(__to_swnode_fwnode) ? \ + container_of(__to_swnode_fwnode, \ + struct swnode, fwnode) : NULL; \ + }) + +static inline struct swnode *dev_to_swnode(struct device *dev) +{ + struct fwnode_handle *fwnode = dev_fwnode(dev); + + if (!fwnode) + return NULL; + + if (!is_software_node(fwnode)) + fwnode = fwnode->secondary; + + return to_swnode(fwnode); +} + +static struct swnode * +software_node_to_swnode(const struct software_node *node) +{ + struct swnode *swnode = NULL; + struct kobject *k; + + if (!node) + return NULL; + + spin_lock(&swnode_kset->list_lock); + + list_for_each_entry(k, &swnode_kset->list, entry) { + swnode = kobj_to_swnode(k); + if (swnode->node == node) + break; + swnode = NULL; + } + + spin_unlock(&swnode_kset->list_lock); + + return swnode; +} + +const struct software_node *to_software_node(const struct fwnode_handle *fwnode) +{ + const struct swnode *swnode = to_swnode(fwnode); + + return swnode ? swnode->node : NULL; +} +EXPORT_SYMBOL_GPL(to_software_node); + +struct fwnode_handle *software_node_fwnode(const struct software_node *node) +{ + struct swnode *swnode = software_node_to_swnode(node); + + return swnode ? &swnode->fwnode : NULL; +} +EXPORT_SYMBOL_GPL(software_node_fwnode); + +/* -------------------------------------------------------------------------- */ +/* property_entry processing */ + +static const struct property_entry * +property_entry_get(const struct property_entry *prop, const char *name) +{ + if (!prop) + return NULL; + + for (; prop->name; prop++) + if (!strcmp(name, prop->name)) + return prop; + + return NULL; +} + +static const void *property_get_pointer(const struct property_entry *prop) +{ + if (!prop->length) + return NULL; + + return prop->is_inline ? &prop->value : prop->pointer; +} + +static const void *property_entry_find(const struct property_entry *props, + const char *propname, size_t length) +{ + const struct property_entry *prop; + const void *pointer; + + prop = property_entry_get(props, propname); + if (!prop) + return ERR_PTR(-EINVAL); + pointer = property_get_pointer(prop); + if (!pointer) + return ERR_PTR(-ENODATA); + if (length > prop->length) + return ERR_PTR(-EOVERFLOW); + return pointer; +} + +static int +property_entry_count_elems_of_size(const struct property_entry *props, + const char *propname, size_t length) +{ + const struct property_entry *prop; + + prop = property_entry_get(props, propname); + if (!prop) + return -EINVAL; + + return prop->length / length; +} + +static int property_entry_read_int_array(const struct property_entry *props, + const char *name, + unsigned int elem_size, void *val, + size_t nval) +{ + const void *pointer; + size_t length; + + if (!val) + return property_entry_count_elems_of_size(props, name, + elem_size); + + if (!is_power_of_2(elem_size) || elem_size > sizeof(u64)) + return -ENXIO; + + length = nval * elem_size; + + pointer = property_entry_find(props, name, length); + if (IS_ERR(pointer)) + return PTR_ERR(pointer); + + memcpy(val, pointer, length); + return 0; +} + +static int property_entry_read_string_array(const struct property_entry *props, + const char *propname, + const char **strings, size_t nval) +{ + const void *pointer; + size_t length; + int array_len; + + /* Find out the array length. */ + array_len = property_entry_count_elems_of_size(props, propname, + sizeof(const char *)); + if (array_len < 0) + return array_len; + + /* Return how many there are if strings is NULL. */ + if (!strings) + return array_len; + + array_len = min_t(size_t, nval, array_len); + length = array_len * sizeof(*strings); + + pointer = property_entry_find(props, propname, length); + if (IS_ERR(pointer)) + return PTR_ERR(pointer); + + memcpy(strings, pointer, length); + + return array_len; +} + +static void property_entry_free_data(const struct property_entry *p) +{ + const char * const *src_str; + size_t i, nval; + + if (p->type == DEV_PROP_STRING) { + src_str = property_get_pointer(p); + nval = p->length / sizeof(*src_str); + for (i = 0; i < nval; i++) + kfree(src_str[i]); + } + + if (!p->is_inline) + kfree(p->pointer); + + kfree(p->name); +} + +static bool property_copy_string_array(const char **dst_ptr, + const char * const *src_ptr, + size_t nval) +{ + int i; + + for (i = 0; i < nval; i++) { + dst_ptr[i] = kstrdup(src_ptr[i], GFP_KERNEL); + if (!dst_ptr[i] && src_ptr[i]) { + while (--i >= 0) + kfree(dst_ptr[i]); + return false; + } + } + + return true; +} + +static int property_entry_copy_data(struct property_entry *dst, + const struct property_entry *src) +{ + const void *pointer = property_get_pointer(src); + void *dst_ptr; + size_t nval; + + /* + * Properties with no data should not be marked as stored + * out of line. + */ + if (!src->is_inline && !src->length) + return -ENODATA; + + /* + * Reference properties are never stored inline as + * they are too big. + */ + if (src->type == DEV_PROP_REF && src->is_inline) + return -EINVAL; + + if (src->length <= sizeof(dst->value)) { + dst_ptr = &dst->value; + dst->is_inline = true; + } else { + dst_ptr = kmalloc(src->length, GFP_KERNEL); + if (!dst_ptr) + return -ENOMEM; + dst->pointer = dst_ptr; + } + + if (src->type == DEV_PROP_STRING) { + nval = src->length / sizeof(const char *); + if (!property_copy_string_array(dst_ptr, pointer, nval)) { + if (!dst->is_inline) + kfree(dst->pointer); + return -ENOMEM; + } + } else { + memcpy(dst_ptr, pointer, src->length); + } + + dst->length = src->length; + dst->type = src->type; + dst->name = kstrdup(src->name, GFP_KERNEL); + if (!dst->name) { + property_entry_free_data(dst); + return -ENOMEM; + } + + return 0; +} + +/** + * property_entries_dup - duplicate array of properties + * @properties: array of properties to copy + * + * This function creates a deep copy of the given NULL-terminated array + * of property entries. + */ +struct property_entry * +property_entries_dup(const struct property_entry *properties) +{ + struct property_entry *p; + int i, n = 0; + int ret; + + if (!properties) + return NULL; + + while (properties[n].name) + n++; + + p = kcalloc(n + 1, sizeof(*p), GFP_KERNEL); + if (!p) + return ERR_PTR(-ENOMEM); + + for (i = 0; i < n; i++) { + ret = property_entry_copy_data(&p[i], &properties[i]); + if (ret) { + while (--i >= 0) + property_entry_free_data(&p[i]); + kfree(p); + return ERR_PTR(ret); + } + } + + return p; +} +EXPORT_SYMBOL_GPL(property_entries_dup); + +/** + * property_entries_free - free previously allocated array of properties + * @properties: array of properties to destroy + * + * This function frees given NULL-terminated array of property entries, + * along with their data. + */ +void property_entries_free(const struct property_entry *properties) +{ + const struct property_entry *p; + + if (!properties) + return; + + for (p = properties; p->name; p++) + property_entry_free_data(p); + + kfree(properties); +} +EXPORT_SYMBOL_GPL(property_entries_free); + +/* -------------------------------------------------------------------------- */ +/* fwnode operations */ + +static struct fwnode_handle *software_node_get(struct fwnode_handle *fwnode) +{ + struct swnode *swnode = to_swnode(fwnode); + + kobject_get(&swnode->kobj); + + return &swnode->fwnode; +} + +static void software_node_put(struct fwnode_handle *fwnode) +{ + struct swnode *swnode = to_swnode(fwnode); + + kobject_put(&swnode->kobj); +} + +static bool software_node_property_present(const struct fwnode_handle *fwnode, + const char *propname) +{ + struct swnode *swnode = to_swnode(fwnode); + + return !!property_entry_get(swnode->node->properties, propname); +} + +static int software_node_read_int_array(const struct fwnode_handle *fwnode, + const char *propname, + unsigned int elem_size, void *val, + size_t nval) +{ + struct swnode *swnode = to_swnode(fwnode); + + return property_entry_read_int_array(swnode->node->properties, propname, + elem_size, val, nval); +} + +static int software_node_read_string_array(const struct fwnode_handle *fwnode, + const char *propname, + const char **val, size_t nval) +{ + struct swnode *swnode = to_swnode(fwnode); + + return property_entry_read_string_array(swnode->node->properties, + propname, val, nval); +} + +static const char * +software_node_get_name(const struct fwnode_handle *fwnode) +{ + const struct swnode *swnode = to_swnode(fwnode); + + return kobject_name(&swnode->kobj); +} + +static const char * +software_node_get_name_prefix(const struct fwnode_handle *fwnode) +{ + struct fwnode_handle *parent; + const char *prefix; + + parent = fwnode_get_parent(fwnode); + if (!parent) + return ""; + + /* Figure out the prefix from the parents. */ + while (is_software_node(parent)) + parent = fwnode_get_next_parent(parent); + + prefix = fwnode_get_name_prefix(parent); + fwnode_handle_put(parent); + + /* Guess something if prefix was NULL. */ + return prefix ?: "/"; +} + +static struct fwnode_handle * +software_node_get_parent(const struct fwnode_handle *fwnode) +{ + struct swnode *swnode = to_swnode(fwnode); + + if (!swnode || !swnode->parent) + return NULL; + + return fwnode_handle_get(&swnode->parent->fwnode); +} + +static struct fwnode_handle * +software_node_get_next_child(const struct fwnode_handle *fwnode, + struct fwnode_handle *child) +{ + struct swnode *p = to_swnode(fwnode); + struct swnode *c = to_swnode(child); + + if (!p || list_empty(&p->children) || + (c && list_is_last(&c->entry, &p->children))) { + fwnode_handle_put(child); + return NULL; + } + + if (c) + c = list_next_entry(c, entry); + else + c = list_first_entry(&p->children, struct swnode, entry); + + fwnode_handle_put(child); + return fwnode_handle_get(&c->fwnode); +} + +static struct fwnode_handle * +software_node_get_named_child_node(const struct fwnode_handle *fwnode, + const char *childname) +{ + struct swnode *swnode = to_swnode(fwnode); + struct swnode *child; + + if (!swnode || list_empty(&swnode->children)) + return NULL; + + list_for_each_entry(child, &swnode->children, entry) { + if (!strcmp(childname, kobject_name(&child->kobj))) { + kobject_get(&child->kobj); + return &child->fwnode; + } + } + return NULL; +} + +static int +software_node_get_reference_args(const struct fwnode_handle *fwnode, + const char *propname, const char *nargs_prop, + unsigned int nargs, unsigned int index, + struct fwnode_reference_args *args) +{ + struct swnode *swnode = to_swnode(fwnode); + const struct software_node_ref_args *ref_array; + const struct software_node_ref_args *ref; + const struct property_entry *prop; + struct fwnode_handle *refnode; + u32 nargs_prop_val; + int error; + int i; + + prop = property_entry_get(swnode->node->properties, propname); + if (!prop) + return -ENOENT; + + if (prop->type != DEV_PROP_REF) + return -EINVAL; + + /* + * We expect that references are never stored inline, even + * single ones, as they are too big. + */ + if (prop->is_inline) + return -EINVAL; + + if ((index + 1) * sizeof(*ref) > prop->length) + return -ENOENT; + + ref_array = prop->pointer; + ref = &ref_array[index]; + + /* + * A software node can reference other software nodes or firmware + * nodes (which are the abstraction layer sitting on top of them). + * This is done to ensure we can create references to static software + * nodes before they're registered with the firmware node framework. + * At the time the reference is being resolved, we expect the swnodes + * in question to already have been registered and to be backed by + * a firmware node. This is why we use the fwnode API below to read the + * relevant properties and bump the reference count. + */ + + if (ref->swnode) + refnode = software_node_fwnode(ref->swnode); + else if (ref->fwnode) + refnode = ref->fwnode; + else + return -EINVAL; + + if (!refnode) + return -ENOENT; + + if (nargs_prop) { + error = fwnode_property_read_u32(refnode, nargs_prop, &nargs_prop_val); + if (error) + return error; + + nargs = nargs_prop_val; + } + + if (nargs > NR_FWNODE_REFERENCE_ARGS) + return -EINVAL; + + if (!args) + return 0; + + args->fwnode = fwnode_handle_get(refnode); + args->nargs = nargs; + + for (i = 0; i < nargs; i++) + args->args[i] = ref->args[i]; + + return 0; +} + +static struct fwnode_handle * +swnode_graph_find_next_port(const struct fwnode_handle *parent, + struct fwnode_handle *port) +{ + struct fwnode_handle *old = port; + + while ((port = software_node_get_next_child(parent, old))) { + /* + * fwnode ports have naming style "port@", so we search for any + * children that follow that convention. + */ + if (!strncmp(to_swnode(port)->node->name, "port@", + strlen("port@"))) + return port; + old = port; + } + + return NULL; +} + +static struct fwnode_handle * +software_node_graph_get_next_endpoint(const struct fwnode_handle *fwnode, + struct fwnode_handle *endpoint) +{ + struct swnode *swnode = to_swnode(fwnode); + struct fwnode_handle *parent; + struct fwnode_handle *port; + + if (!swnode) + return NULL; + + if (endpoint) { + port = software_node_get_parent(endpoint); + parent = software_node_get_parent(port); + } else { + parent = software_node_get_named_child_node(fwnode, "ports"); + if (!parent) + parent = software_node_get(&swnode->fwnode); + + port = swnode_graph_find_next_port(parent, NULL); + } + + for (; port; port = swnode_graph_find_next_port(parent, port)) { + endpoint = software_node_get_next_child(port, endpoint); + if (endpoint) { + fwnode_handle_put(port); + break; + } + } + + fwnode_handle_put(parent); + + return endpoint; +} + +static struct fwnode_handle * +software_node_graph_get_remote_endpoint(const struct fwnode_handle *fwnode) +{ + struct swnode *swnode = to_swnode(fwnode); + const struct software_node_ref_args *ref; + const struct property_entry *prop; + + if (!swnode) + return NULL; + + prop = property_entry_get(swnode->node->properties, "remote-endpoint"); + if (!prop || prop->type != DEV_PROP_REF || prop->is_inline) + return NULL; + + ref = prop->pointer; + + if (!ref->swnode) + return NULL; + + return software_node_get(software_node_fwnode(ref->swnode)); +} + +static struct fwnode_handle * +software_node_graph_get_port_parent(struct fwnode_handle *fwnode) +{ + struct swnode *swnode = to_swnode(fwnode); + + swnode = swnode->parent; + if (swnode && !strcmp(swnode->node->name, "ports")) + swnode = swnode->parent; + + return swnode ? software_node_get(&swnode->fwnode) : NULL; +} + +static int +software_node_graph_parse_endpoint(const struct fwnode_handle *fwnode, + struct fwnode_endpoint *endpoint) +{ + struct swnode *swnode = to_swnode(fwnode); + const char *parent_name = swnode->parent->node->name; + int ret; + + if (strlen("port@") >= strlen(parent_name) || + strncmp(parent_name, "port@", strlen("port@"))) + return -EINVAL; + + /* Ports have naming style "port@n", we need to select the n */ + ret = kstrtou32(parent_name + strlen("port@"), 10, &endpoint->port); + if (ret) + return ret; + + endpoint->id = swnode->id; + endpoint->local_fwnode = fwnode; + + return 0; +} + +static const struct fwnode_operations software_node_ops = { + .get = software_node_get, + .put = software_node_put, + .property_present = software_node_property_present, + .property_read_bool = software_node_property_present, + .property_read_int_array = software_node_read_int_array, + .property_read_string_array = software_node_read_string_array, + .get_name = software_node_get_name, + .get_name_prefix = software_node_get_name_prefix, + .get_parent = software_node_get_parent, + .get_next_child_node = software_node_get_next_child, + .get_named_child_node = software_node_get_named_child_node, + .get_reference_args = software_node_get_reference_args, + .graph_get_next_endpoint = software_node_graph_get_next_endpoint, + .graph_get_remote_endpoint = software_node_graph_get_remote_endpoint, + .graph_get_port_parent = software_node_graph_get_port_parent, + .graph_parse_endpoint = software_node_graph_parse_endpoint, +}; + +/* -------------------------------------------------------------------------- */ + +/** + * software_node_find_by_name - Find software node by name + * @parent: Parent of the software node + * @name: Name of the software node + * + * The function will find a node that is child of @parent and that is named + * @name. If no node is found, the function returns NULL. + * + * NOTE: you will need to drop the reference with fwnode_handle_put() after use. + */ +const struct software_node * +software_node_find_by_name(const struct software_node *parent, const char *name) +{ + struct swnode *swnode = NULL; + struct kobject *k; + + if (!name) + return NULL; + + spin_lock(&swnode_kset->list_lock); + + list_for_each_entry(k, &swnode_kset->list, entry) { + swnode = kobj_to_swnode(k); + if (parent == swnode->node->parent && swnode->node->name && + !strcmp(name, swnode->node->name)) { + kobject_get(&swnode->kobj); + break; + } + swnode = NULL; + } + + spin_unlock(&swnode_kset->list_lock); + + return swnode ? swnode->node : NULL; +} +EXPORT_SYMBOL_GPL(software_node_find_by_name); + +static struct software_node *software_node_alloc(const struct property_entry *properties) +{ + struct property_entry *props; + struct software_node *node; + + props = property_entries_dup(properties); + if (IS_ERR(props)) + return ERR_CAST(props); + + node = kzalloc(sizeof(*node), GFP_KERNEL); + if (!node) { + property_entries_free(props); + return ERR_PTR(-ENOMEM); + } + + node->properties = props; + + return node; +} + +static void software_node_free(const struct software_node *node) +{ + property_entries_free(node->properties); + kfree(node); +} + +static void software_node_release(struct kobject *kobj) +{ + struct swnode *swnode = kobj_to_swnode(kobj); + + if (swnode->parent) { + ida_free(&swnode->parent->child_ids, swnode->id); + list_del(&swnode->entry); + } else { + ida_free(&swnode_root_ids, swnode->id); + } + + if (swnode->allocated) + software_node_free(swnode->node); + + ida_destroy(&swnode->child_ids); + kfree(swnode); +} + +static const struct kobj_type software_node_type = { + .release = software_node_release, + .sysfs_ops = &kobj_sysfs_ops, +}; + +static struct fwnode_handle * +swnode_register(const struct software_node *node, struct swnode *parent, + unsigned int allocated) +{ + struct swnode *swnode; + int ret; + + swnode = kzalloc(sizeof(*swnode), GFP_KERNEL); + if (!swnode) + return ERR_PTR(-ENOMEM); + + ret = ida_alloc(parent ? &parent->child_ids : &swnode_root_ids, + GFP_KERNEL); + if (ret < 0) { + kfree(swnode); + return ERR_PTR(ret); + } + + swnode->id = ret; + swnode->node = node; + swnode->parent = parent; + swnode->kobj.kset = swnode_kset; + fwnode_init(&swnode->fwnode, &software_node_ops); + + ida_init(&swnode->child_ids); + INIT_LIST_HEAD(&swnode->entry); + INIT_LIST_HEAD(&swnode->children); + + if (node->name) + ret = kobject_init_and_add(&swnode->kobj, &software_node_type, + parent ? &parent->kobj : NULL, + "%s", node->name); + else + ret = kobject_init_and_add(&swnode->kobj, &software_node_type, + parent ? &parent->kobj : NULL, + "node%d", swnode->id); + if (ret) { + kobject_put(&swnode->kobj); + return ERR_PTR(ret); + } + + /* + * Assign the flag only in the successful case, so + * the above kobject_put() won't mess up with properties. + */ + swnode->allocated = allocated; + + if (parent) + list_add_tail(&swnode->entry, &parent->children); + + kobject_uevent(&swnode->kobj, KOBJ_ADD); + return &swnode->fwnode; +} + +/** + * software_node_register_node_group - Register a group of software nodes + * @node_group: NULL terminated array of software node pointers to be registered + * + * Register multiple software nodes at once. If any node in the array + * has its .parent pointer set (which can only be to another software_node), + * then its parent **must** have been registered before it is; either outside + * of this function or by ordering the array such that parent comes before + * child. + */ +int software_node_register_node_group(const struct software_node * const *node_group) +{ + unsigned int i; + int ret; + + if (!node_group) + return 0; + + for (i = 0; node_group[i]; i++) { + ret = software_node_register(node_group[i]); + if (ret) { + software_node_unregister_node_group(node_group); + return ret; + } + } + + return 0; +} +EXPORT_SYMBOL_GPL(software_node_register_node_group); + +/** + * software_node_unregister_node_group - Unregister a group of software nodes + * @node_group: NULL terminated array of software node pointers to be unregistered + * + * Unregister multiple software nodes at once. If parent pointers are set up + * in any of the software nodes then the array **must** be ordered such that + * parents come before their children. + * + * NOTE: If you are uncertain whether the array is ordered such that + * parents will be unregistered before their children, it is wiser to + * remove the nodes individually, in the correct order (child before + * parent). + */ +void software_node_unregister_node_group(const struct software_node * const *node_group) +{ + unsigned int i = 0; + + if (!node_group) + return; + + while (node_group[i]) + i++; + + while (i--) + software_node_unregister(node_group[i]); +} +EXPORT_SYMBOL_GPL(software_node_unregister_node_group); + +/** + * software_node_register - Register static software node + * @node: The software node to be registered + */ +int software_node_register(const struct software_node *node) +{ + struct swnode *parent = software_node_to_swnode(node->parent); + + if (software_node_to_swnode(node)) + return -EEXIST; + + if (node->parent && !parent) + return -EINVAL; + + return PTR_ERR_OR_ZERO(swnode_register(node, parent, 0)); +} +EXPORT_SYMBOL_GPL(software_node_register); + +/** + * software_node_unregister - Unregister static software node + * @node: The software node to be unregistered + */ +void software_node_unregister(const struct software_node *node) +{ + struct swnode *swnode; + + swnode = software_node_to_swnode(node); + if (swnode) + fwnode_remove_software_node(&swnode->fwnode); +} +EXPORT_SYMBOL_GPL(software_node_unregister); + +struct fwnode_handle * +fwnode_create_software_node(const struct property_entry *properties, + const struct fwnode_handle *parent) +{ + struct fwnode_handle *fwnode; + struct software_node *node; + struct swnode *p; + + if (IS_ERR(parent)) + return ERR_CAST(parent); + + p = to_swnode(parent); + if (parent && !p) + return ERR_PTR(-EINVAL); + + node = software_node_alloc(properties); + if (IS_ERR(node)) + return ERR_CAST(node); + + node->parent = p ? p->node : NULL; + + fwnode = swnode_register(node, p, 1); + if (IS_ERR(fwnode)) + software_node_free(node); + + return fwnode; +} +EXPORT_SYMBOL_GPL(fwnode_create_software_node); + +void fwnode_remove_software_node(struct fwnode_handle *fwnode) +{ + struct swnode *swnode = to_swnode(fwnode); + + if (!swnode) + return; + + kobject_put(&swnode->kobj); +} +EXPORT_SYMBOL_GPL(fwnode_remove_software_node); + +/** + * device_add_software_node - Assign software node to a device + * @dev: The device the software node is meant for. + * @node: The software node. + * + * This function will make @node the secondary firmware node pointer of @dev. If + * @dev has no primary node, then @node will become the primary node. The + * function will register @node automatically if it wasn't already registered. + */ +int device_add_software_node(struct device *dev, const struct software_node *node) +{ + struct swnode *swnode; + int ret; + + /* Only one software node per device. */ + if (dev_to_swnode(dev)) + return -EBUSY; + + swnode = software_node_to_swnode(node); + if (swnode) { + kobject_get(&swnode->kobj); + } else { + ret = software_node_register(node); + if (ret) + return ret; + + swnode = software_node_to_swnode(node); + } + + set_secondary_fwnode(dev, &swnode->fwnode); + + /* + * If the device has been fully registered by the time this function is + * called, software_node_notify() must be called separately so that the + * symlinks get created and the reference count of the node is kept in + * balance. + */ + if (device_is_registered(dev)) + software_node_notify(dev); + + return 0; +} +EXPORT_SYMBOL_GPL(device_add_software_node); + +/** + * device_remove_software_node - Remove device's software node + * @dev: The device with the software node. + * + * This function will unregister the software node of @dev. + */ +void device_remove_software_node(struct device *dev) +{ + struct swnode *swnode; + + swnode = dev_to_swnode(dev); + if (!swnode) + return; + + if (device_is_registered(dev)) + software_node_notify_remove(dev); + + set_secondary_fwnode(dev, NULL); + kobject_put(&swnode->kobj); +} +EXPORT_SYMBOL_GPL(device_remove_software_node); + +/** + * device_create_managed_software_node - Create a software node for a device + * @dev: The device the software node is assigned to. + * @properties: Device properties for the software node. + * @parent: Parent of the software node. + * + * Creates a software node as a managed resource for @dev, which means the + * lifetime of the newly created software node is tied to the lifetime of @dev. + * Software nodes created with this function should not be reused or shared + * because of that. The function takes a deep copy of @properties for the + * software node. + * + * Since the new software node is assigned directly to @dev, and since it should + * not be shared, it is not returned to the caller. The function returns 0 on + * success, and errno in case of an error. + */ +int device_create_managed_software_node(struct device *dev, + const struct property_entry *properties, + const struct software_node *parent) +{ + struct fwnode_handle *p = software_node_fwnode(parent); + struct fwnode_handle *fwnode; + + if (parent && !p) + return -EINVAL; + + fwnode = fwnode_create_software_node(properties, p); + if (IS_ERR(fwnode)) + return PTR_ERR(fwnode); + + to_swnode(fwnode)->managed = true; + set_secondary_fwnode(dev, fwnode); + + if (device_is_registered(dev)) + software_node_notify(dev); + + return 0; +} +EXPORT_SYMBOL_GPL(device_create_managed_software_node); + +void software_node_notify(struct device *dev) +{ + struct swnode *swnode; + int ret; + + swnode = dev_to_swnode(dev); + if (!swnode) + return; + + kobject_get(&swnode->kobj); + ret = sysfs_create_link(&dev->kobj, &swnode->kobj, "software_node"); + if (ret) + return; + + ret = sysfs_create_link(&swnode->kobj, &dev->kobj, dev_name(dev)); + if (ret) { + sysfs_remove_link(&dev->kobj, "software_node"); + return; + } +} + +void software_node_notify_remove(struct device *dev) +{ + struct swnode *swnode; + + swnode = dev_to_swnode(dev); + if (!swnode) + return; + + sysfs_remove_link(&swnode->kobj, dev_name(dev)); + sysfs_remove_link(&dev->kobj, "software_node"); + kobject_put(&swnode->kobj); + + if (swnode->managed) { + set_secondary_fwnode(dev, NULL); + kobject_put(&swnode->kobj); + } +} + +static int __init software_node_init(void) +{ + swnode_kset = kset_create_and_add("software_nodes", NULL, kernel_kobj); + if (!swnode_kset) + return -ENOMEM; + return 0; +} +postcore_initcall(software_node_init); + +static void __exit software_node_exit(void) +{ + ida_destroy(&swnode_root_ids); + kset_unregister(swnode_kset); +} +__exitcall(software_node_exit); diff --git a/drivers/base/syscore.c b/drivers/base/syscore.c index e8d11b6630ee..483adb796654 100644 --- a/drivers/base/syscore.c +++ b/drivers/base/syscore.c @@ -1,42 +1,42 @@ +// SPDX-License-Identifier: GPL-2.0 /* * syscore.c - Execution of system core operations. * * Copyright (C) 2011 Rafael J. Wysocki <rjw@sisk.pl>, Novell Inc. - * - * This file is released under the GPLv2. */ #include <linux/syscore_ops.h> #include <linux/mutex.h> #include <linux/module.h> -#include <linux/interrupt.h> +#include <linux/suspend.h> +#include <trace/events/power.h> -static LIST_HEAD(syscore_ops_list); -static DEFINE_MUTEX(syscore_ops_lock); +static LIST_HEAD(syscore_list); +static DEFINE_MUTEX(syscore_lock); /** - * register_syscore_ops - Register a set of system core operations. - * @ops: System core operations to register. + * register_syscore - Register a set of system core operations. + * @syscore: System core operations to register. */ -void register_syscore_ops(struct syscore_ops *ops) +void register_syscore(struct syscore *syscore) { - mutex_lock(&syscore_ops_lock); - list_add_tail(&ops->node, &syscore_ops_list); - mutex_unlock(&syscore_ops_lock); + mutex_lock(&syscore_lock); + list_add_tail(&syscore->node, &syscore_list); + mutex_unlock(&syscore_lock); } -EXPORT_SYMBOL_GPL(register_syscore_ops); +EXPORT_SYMBOL_GPL(register_syscore); /** - * unregister_syscore_ops - Unregister a set of system core operations. - * @ops: System core operations to unregister. + * unregister_syscore - Unregister a set of system core operations. + * @syscore: System core operations to unregister. */ -void unregister_syscore_ops(struct syscore_ops *ops) +void unregister_syscore(struct syscore *syscore) { - mutex_lock(&syscore_ops_lock); - list_del(&ops->node); - mutex_unlock(&syscore_ops_lock); + mutex_lock(&syscore_lock); + list_del(&syscore->node); + mutex_unlock(&syscore_lock); } -EXPORT_SYMBOL_GPL(unregister_syscore_ops); +EXPORT_SYMBOL_GPL(unregister_syscore); #ifdef CONFIG_PM_SLEEP /** @@ -46,38 +46,40 @@ EXPORT_SYMBOL_GPL(unregister_syscore_ops); */ int syscore_suspend(void) { - struct syscore_ops *ops; + struct syscore *syscore; int ret = 0; - pr_debug("Checking wakeup interrupts\n"); + trace_suspend_resume(TPS("syscore_suspend"), 0, true); + pm_pr_dbg("Checking wakeup interrupts\n"); /* Return error code if there are any wakeup interrupts pending. */ - ret = check_wakeup_irqs(); - if (ret) - return ret; + if (pm_wakeup_pending()) + return -EBUSY; WARN_ONCE(!irqs_disabled(), "Interrupts enabled before system core suspend.\n"); - list_for_each_entry_reverse(ops, &syscore_ops_list, node) - if (ops->suspend) { - if (initcall_debug) - pr_info("PM: Calling %pF\n", ops->suspend); - ret = ops->suspend(); + list_for_each_entry_reverse(syscore, &syscore_list, node) + if (syscore->ops->suspend) { + pm_pr_dbg("Calling %pS\n", syscore->ops->suspend); + ret = syscore->ops->suspend(syscore->data); if (ret) goto err_out; WARN_ONCE(!irqs_disabled(), - "Interrupts enabled after %pF\n", ops->suspend); + "Interrupts enabled after %pS\n", + syscore->ops->suspend); } + trace_suspend_resume(TPS("syscore_suspend"), 0, false); return 0; err_out: - pr_err("PM: System core suspend callback %pF failed.\n", ops->suspend); + pr_err("PM: System core suspend callback %pS failed.\n", + syscore->ops->suspend); - list_for_each_entry_continue(ops, &syscore_ops_list, node) - if (ops->resume) - ops->resume(); + list_for_each_entry_continue(syscore, &syscore_list, node) + if (syscore->ops->resume) + syscore->ops->resume(syscore->data); return ret; } @@ -90,19 +92,21 @@ EXPORT_SYMBOL_GPL(syscore_suspend); */ void syscore_resume(void) { - struct syscore_ops *ops; + struct syscore *syscore; + trace_suspend_resume(TPS("syscore_resume"), 0, true); WARN_ONCE(!irqs_disabled(), "Interrupts enabled before system core resume.\n"); - list_for_each_entry(ops, &syscore_ops_list, node) - if (ops->resume) { - if (initcall_debug) - pr_info("PM: Calling %pF\n", ops->resume); - ops->resume(); + list_for_each_entry(syscore, &syscore_list, node) + if (syscore->ops->resume) { + pm_pr_dbg("Calling %pS\n", syscore->ops->resume); + syscore->ops->resume(syscore->data); WARN_ONCE(!irqs_disabled(), - "Interrupts enabled after %pF\n", ops->resume); + "Interrupts enabled after %pS\n", + syscore->ops->resume); } + trace_suspend_resume(TPS("syscore_resume"), 0, false); } EXPORT_SYMBOL_GPL(syscore_resume); #endif /* CONFIG_PM_SLEEP */ @@ -112,16 +116,17 @@ EXPORT_SYMBOL_GPL(syscore_resume); */ void syscore_shutdown(void) { - struct syscore_ops *ops; + struct syscore *syscore; - mutex_lock(&syscore_ops_lock); + mutex_lock(&syscore_lock); - list_for_each_entry_reverse(ops, &syscore_ops_list, node) - if (ops->shutdown) { + list_for_each_entry_reverse(syscore, &syscore_list, node) + if (syscore->ops->shutdown) { if (initcall_debug) - pr_info("PM: Calling %pF\n", ops->shutdown); - ops->shutdown(); + pr_info("PM: Calling %pS\n", + syscore->ops->shutdown); + syscore->ops->shutdown(syscore->data); } - mutex_unlock(&syscore_ops_lock); + mutex_unlock(&syscore_lock); } diff --git a/drivers/base/test/.kunitconfig b/drivers/base/test/.kunitconfig new file mode 100644 index 000000000000..473923f0998b --- /dev/null +++ b/drivers/base/test/.kunitconfig @@ -0,0 +1,2 @@ +CONFIG_KUNIT=y +CONFIG_DM_KUNIT_TEST=y diff --git a/drivers/base/test/Kconfig b/drivers/base/test/Kconfig new file mode 100644 index 000000000000..2756870615cc --- /dev/null +++ b/drivers/base/test/Kconfig @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: GPL-2.0 +config TEST_ASYNC_DRIVER_PROBE + tristate "Build kernel module to test asynchronous driver probing" + depends on m + help + Enabling this option produces a kernel module that allows + testing asynchronous driver probing by the device core. + The module name will be test_async_driver_probe.ko + + If unsure say N. + +config DM_KUNIT_TEST + tristate "KUnit Tests for the device model" if !KUNIT_ALL_TESTS + depends on KUNIT + default KUNIT_ALL_TESTS + +config DRIVER_PE_KUNIT_TEST + tristate "KUnit Tests for property entry API" if !KUNIT_ALL_TESTS + depends on KUNIT + default KUNIT_ALL_TESTS diff --git a/drivers/base/test/Makefile b/drivers/base/test/Makefile new file mode 100644 index 000000000000..e321dfc7e922 --- /dev/null +++ b/drivers/base/test/Makefile @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_TEST_ASYNC_DRIVER_PROBE) += test_async_driver_probe.o + +obj-$(CONFIG_DM_KUNIT_TEST) += root-device-test.o +obj-$(CONFIG_DM_KUNIT_TEST) += platform-device-test.o + +obj-$(CONFIG_DRIVER_PE_KUNIT_TEST) += property-entry-test.o +CFLAGS_property-entry-test.o += $(DISABLE_STRUCTLEAK_PLUGIN) diff --git a/drivers/base/test/platform-device-test.c b/drivers/base/test/platform-device-test.c new file mode 100644 index 000000000000..6355a2231b74 --- /dev/null +++ b/drivers/base/test/platform-device-test.c @@ -0,0 +1,263 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <kunit/platform_device.h> +#include <kunit/resource.h> + +#include <linux/device.h> +#include <linux/device/bus.h> +#include <linux/of_platform.h> +#include <linux/platform_device.h> + +#define DEVICE_NAME "test" + +struct test_priv { + bool probe_done; + bool release_done; + wait_queue_head_t probe_wq; + wait_queue_head_t release_wq; + struct device *dev; +}; + +static int platform_device_devm_init(struct kunit *test) +{ + struct test_priv *priv; + + priv = kunit_kzalloc(test, sizeof(*priv), GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, priv); + init_waitqueue_head(&priv->probe_wq); + init_waitqueue_head(&priv->release_wq); + + test->priv = priv; + + return 0; +} + +static void devm_device_action(void *ptr) +{ + struct test_priv *priv = ptr; + + priv->release_done = true; + wake_up_interruptible(&priv->release_wq); +} + +static void devm_put_device_action(void *ptr) +{ + struct test_priv *priv = ptr; + + put_device(priv->dev); + priv->release_done = true; + wake_up_interruptible(&priv->release_wq); +} + +#define RELEASE_TIMEOUT_MS 100 + +/* + * Tests that a platform bus, non-probed device will run its + * device-managed actions when unregistered. + */ +static void platform_device_devm_register_unregister_test(struct kunit *test) +{ + struct platform_device *pdev; + struct test_priv *priv = test->priv; + int ret; + + pdev = platform_device_alloc(DEVICE_NAME, PLATFORM_DEVID_NONE); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pdev); + + ret = platform_device_add(pdev); + KUNIT_ASSERT_EQ(test, ret, 0); + + priv->dev = &pdev->dev; + + ret = devm_add_action_or_reset(priv->dev, devm_device_action, priv); + KUNIT_ASSERT_EQ(test, ret, 0); + + platform_device_unregister(pdev); + + ret = wait_event_interruptible_timeout(priv->release_wq, priv->release_done, + msecs_to_jiffies(RELEASE_TIMEOUT_MS)); + KUNIT_EXPECT_GT(test, ret, 0); +} + +/* + * Tests that a platform bus, non-probed device will run its + * device-managed actions when unregistered, even if someone still holds + * a reference to it. + */ +static void platform_device_devm_register_get_unregister_with_devm_test(struct kunit *test) +{ + struct platform_device *pdev; + struct test_priv *priv = test->priv; + int ret; + + pdev = platform_device_alloc(DEVICE_NAME, PLATFORM_DEVID_NONE); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pdev); + + ret = platform_device_add(pdev); + KUNIT_ASSERT_EQ(test, ret, 0); + + priv->dev = &pdev->dev; + + get_device(priv->dev); + + ret = devm_add_action_or_reset(priv->dev, devm_put_device_action, priv); + KUNIT_ASSERT_EQ(test, ret, 0); + + platform_device_unregister(pdev); + + ret = wait_event_interruptible_timeout(priv->release_wq, priv->release_done, + msecs_to_jiffies(RELEASE_TIMEOUT_MS)); + KUNIT_EXPECT_GT(test, ret, 0); +} + +static int fake_probe(struct platform_device *pdev) +{ + struct test_priv *priv = platform_get_drvdata(pdev); + + priv->probe_done = true; + wake_up_interruptible(&priv->probe_wq); + + return 0; +} + +static struct platform_driver fake_driver = { + .probe = fake_probe, + .driver = { + .name = DEVICE_NAME, + }, +}; + +/* + * Tests that a platform bus, probed device will run its device-managed + * actions when unregistered. + */ +static void probed_platform_device_devm_register_unregister_test(struct kunit *test) +{ + struct platform_device *pdev; + struct test_priv *priv = test->priv; + int ret; + + ret = platform_driver_register(&fake_driver); + KUNIT_ASSERT_EQ(test, ret, 0); + + pdev = platform_device_alloc(DEVICE_NAME, PLATFORM_DEVID_NONE); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pdev); + + priv->dev = &pdev->dev; + platform_set_drvdata(pdev, priv); + + ret = platform_device_add(pdev); + KUNIT_ASSERT_EQ(test, ret, 0); + + ret = wait_event_interruptible_timeout(priv->probe_wq, priv->probe_done, + msecs_to_jiffies(RELEASE_TIMEOUT_MS)); + KUNIT_ASSERT_GT(test, ret, 0); + + ret = devm_add_action_or_reset(priv->dev, devm_device_action, priv); + KUNIT_ASSERT_EQ(test, ret, 0); + + platform_device_unregister(pdev); + + ret = wait_event_interruptible_timeout(priv->release_wq, priv->release_done, + msecs_to_jiffies(RELEASE_TIMEOUT_MS)); + KUNIT_EXPECT_GT(test, ret, 0); + + platform_driver_unregister(&fake_driver); +} + +/* + * Tests that a platform bus, probed device will run its device-managed + * actions when unregistered, even if someone still holds a reference to + * it. + */ +static void probed_platform_device_devm_register_get_unregister_with_devm_test(struct kunit *test) +{ + struct platform_device *pdev; + struct test_priv *priv = test->priv; + int ret; + + ret = platform_driver_register(&fake_driver); + KUNIT_ASSERT_EQ(test, ret, 0); + + pdev = platform_device_alloc(DEVICE_NAME, PLATFORM_DEVID_NONE); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pdev); + + priv->dev = &pdev->dev; + platform_set_drvdata(pdev, priv); + + ret = platform_device_add(pdev); + KUNIT_ASSERT_EQ(test, ret, 0); + + ret = wait_event_interruptible_timeout(priv->probe_wq, priv->probe_done, + msecs_to_jiffies(RELEASE_TIMEOUT_MS)); + KUNIT_ASSERT_GT(test, ret, 0); + + get_device(priv->dev); + + ret = devm_add_action_or_reset(priv->dev, devm_put_device_action, priv); + KUNIT_ASSERT_EQ(test, ret, 0); + + platform_device_unregister(pdev); + + ret = wait_event_interruptible_timeout(priv->release_wq, priv->release_done, + msecs_to_jiffies(RELEASE_TIMEOUT_MS)); + KUNIT_EXPECT_GT(test, ret, 0); + + platform_driver_unregister(&fake_driver); +} + +static struct kunit_case platform_device_devm_tests[] = { + KUNIT_CASE(platform_device_devm_register_unregister_test), + KUNIT_CASE(platform_device_devm_register_get_unregister_with_devm_test), + KUNIT_CASE(probed_platform_device_devm_register_unregister_test), + KUNIT_CASE(probed_platform_device_devm_register_get_unregister_with_devm_test), + {} +}; + +static struct kunit_suite platform_device_devm_test_suite = { + .name = "platform-device-devm", + .init = platform_device_devm_init, + .test_cases = platform_device_devm_tests, +}; + +static void platform_device_find_by_null_test(struct kunit *test) +{ + struct platform_device *pdev; + int ret; + + pdev = kunit_platform_device_alloc(test, DEVICE_NAME, PLATFORM_DEVID_NONE); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pdev); + + ret = kunit_platform_device_add(test, pdev); + KUNIT_ASSERT_EQ(test, ret, 0); + + KUNIT_EXPECT_PTR_EQ(test, of_find_device_by_node(NULL), NULL); + + KUNIT_EXPECT_PTR_EQ(test, bus_find_device_by_of_node(&platform_bus_type, NULL), NULL); + KUNIT_EXPECT_PTR_EQ(test, bus_find_device_by_fwnode(&platform_bus_type, NULL), NULL); + KUNIT_EXPECT_PTR_EQ(test, bus_find_device_by_acpi_dev(&platform_bus_type, NULL), NULL); + + KUNIT_EXPECT_FALSE(test, device_match_of_node(&pdev->dev, NULL)); + KUNIT_EXPECT_FALSE(test, device_match_fwnode(&pdev->dev, NULL)); + KUNIT_EXPECT_FALSE(test, device_match_acpi_dev(&pdev->dev, NULL)); + KUNIT_EXPECT_FALSE(test, device_match_acpi_handle(&pdev->dev, NULL)); +} + +static struct kunit_case platform_device_match_tests[] = { + KUNIT_CASE(platform_device_find_by_null_test), + {} +}; + +static struct kunit_suite platform_device_match_test_suite = { + .name = "platform-device-match", + .test_cases = platform_device_match_tests, +}; + +kunit_test_suites( + &platform_device_devm_test_suite, + &platform_device_match_test_suite, +); + +MODULE_DESCRIPTION("Test module for platform devices"); +MODULE_AUTHOR("Maxime Ripard <mripard@kernel.org>"); +MODULE_LICENSE("GPL"); diff --git a/drivers/base/test/property-entry-test.c b/drivers/base/test/property-entry-test.c new file mode 100644 index 000000000000..a8657eb06f94 --- /dev/null +++ b/drivers/base/test/property-entry-test.c @@ -0,0 +1,512 @@ +// SPDX-License-Identifier: GPL-2.0 +// Unit tests for property entries API +// +// Copyright 2019 Google LLC. + +#include <kunit/test.h> +#include <linux/property.h> +#include <linux/types.h> + +static void pe_test_uints(struct kunit *test) +{ + static const struct property_entry entries[] = { + PROPERTY_ENTRY_U8("prop-u8", 8), + PROPERTY_ENTRY_U16("prop-u16", 16), + PROPERTY_ENTRY_U32("prop-u32", 32), + PROPERTY_ENTRY_U64("prop-u64", 64), + { } + }; + + struct fwnode_handle *node; + u8 val_u8, array_u8[2]; + u16 val_u16, array_u16[2]; + u32 val_u32, array_u32[2]; + u64 val_u64, array_u64[2]; + int error; + + node = fwnode_create_software_node(entries, NULL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, node); + + error = fwnode_property_count_u8(node, "prop-u8"); + KUNIT_EXPECT_EQ(test, error, 1); + + error = fwnode_property_read_u8(node, "prop-u8", &val_u8); + KUNIT_EXPECT_EQ(test, error, 0); + KUNIT_EXPECT_EQ(test, val_u8, 8); + + error = fwnode_property_read_u8_array(node, "prop-u8", array_u8, 1); + KUNIT_EXPECT_EQ(test, error, 0); + KUNIT_EXPECT_EQ(test, array_u8[0], 8); + + error = fwnode_property_read_u8_array(node, "prop-u8", array_u8, 2); + KUNIT_EXPECT_NE(test, error, 0); + + error = fwnode_property_read_u8(node, "no-prop-u8", &val_u8); + KUNIT_EXPECT_NE(test, error, 0); + + error = fwnode_property_read_u8_array(node, "no-prop-u8", array_u8, 1); + KUNIT_EXPECT_NE(test, error, 0); + + error = fwnode_property_read_u16(node, "prop-u16", &val_u16); + KUNIT_EXPECT_EQ(test, error, 0); + KUNIT_EXPECT_EQ(test, val_u16, 16); + + error = fwnode_property_count_u16(node, "prop-u16"); + KUNIT_EXPECT_EQ(test, error, 1); + + error = fwnode_property_read_u16_array(node, "prop-u16", array_u16, 1); + KUNIT_EXPECT_EQ(test, error, 0); + KUNIT_EXPECT_EQ(test, array_u16[0], 16); + + error = fwnode_property_read_u16_array(node, "prop-u16", array_u16, 2); + KUNIT_EXPECT_NE(test, error, 0); + + error = fwnode_property_read_u16(node, "no-prop-u16", &val_u16); + KUNIT_EXPECT_NE(test, error, 0); + + error = fwnode_property_read_u16_array(node, "no-prop-u16", array_u16, 1); + KUNIT_EXPECT_NE(test, error, 0); + + error = fwnode_property_read_u32(node, "prop-u32", &val_u32); + KUNIT_EXPECT_EQ(test, error, 0); + KUNIT_EXPECT_EQ(test, val_u32, 32); + + error = fwnode_property_count_u32(node, "prop-u32"); + KUNIT_EXPECT_EQ(test, error, 1); + + error = fwnode_property_read_u32_array(node, "prop-u32", array_u32, 1); + KUNIT_EXPECT_EQ(test, error, 0); + KUNIT_EXPECT_EQ(test, array_u32[0], 32); + + error = fwnode_property_read_u32_array(node, "prop-u32", array_u32, 2); + KUNIT_EXPECT_NE(test, error, 0); + + error = fwnode_property_read_u32(node, "no-prop-u32", &val_u32); + KUNIT_EXPECT_NE(test, error, 0); + + error = fwnode_property_read_u32_array(node, "no-prop-u32", array_u32, 1); + KUNIT_EXPECT_NE(test, error, 0); + + error = fwnode_property_read_u64(node, "prop-u64", &val_u64); + KUNIT_EXPECT_EQ(test, error, 0); + KUNIT_EXPECT_EQ(test, val_u64, 64); + + error = fwnode_property_count_u64(node, "prop-u64"); + KUNIT_EXPECT_EQ(test, error, 1); + + error = fwnode_property_read_u64_array(node, "prop-u64", array_u64, 1); + KUNIT_EXPECT_EQ(test, error, 0); + KUNIT_EXPECT_EQ(test, array_u64[0], 64); + + error = fwnode_property_read_u64_array(node, "prop-u64", array_u64, 2); + KUNIT_EXPECT_NE(test, error, 0); + + error = fwnode_property_read_u64(node, "no-prop-u64", &val_u64); + KUNIT_EXPECT_NE(test, error, 0); + + error = fwnode_property_read_u64_array(node, "no-prop-u64", array_u64, 1); + KUNIT_EXPECT_NE(test, error, 0); + + /* Count 64-bit values as 16-bit */ + error = fwnode_property_count_u16(node, "prop-u64"); + KUNIT_EXPECT_EQ(test, error, 4); + + fwnode_remove_software_node(node); +} + +static void pe_test_uint_arrays(struct kunit *test) +{ + static const u8 a_u8[10] = { 8, 9 }; + static const u16 a_u16[10] = { 16, 17 }; + static const u32 a_u32[10] = { 32, 33 }; + static const u64 a_u64[10] = { 64, 65 }; + static const struct property_entry entries[] = { + PROPERTY_ENTRY_U8_ARRAY("prop-u8", a_u8), + PROPERTY_ENTRY_U16_ARRAY("prop-u16", a_u16), + PROPERTY_ENTRY_U32_ARRAY("prop-u32", a_u32), + PROPERTY_ENTRY_U64_ARRAY("prop-u64", a_u64), + { } + }; + + struct fwnode_handle *node; + u8 val_u8, array_u8[32]; + u16 val_u16, array_u16[32]; + u32 val_u32, array_u32[32]; + u64 val_u64, array_u64[32]; + int error; + + node = fwnode_create_software_node(entries, NULL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, node); + + error = fwnode_property_read_u8(node, "prop-u8", &val_u8); + KUNIT_EXPECT_EQ(test, error, 0); + KUNIT_EXPECT_EQ(test, val_u8, 8); + + error = fwnode_property_count_u8(node, "prop-u8"); + KUNIT_EXPECT_EQ(test, error, 10); + + error = fwnode_property_read_u8_array(node, "prop-u8", array_u8, 1); + KUNIT_EXPECT_EQ(test, error, 0); + KUNIT_EXPECT_EQ(test, array_u8[0], 8); + + error = fwnode_property_read_u8_array(node, "prop-u8", array_u8, 2); + KUNIT_EXPECT_EQ(test, error, 0); + KUNIT_EXPECT_EQ(test, array_u8[0], 8); + KUNIT_EXPECT_EQ(test, array_u8[1], 9); + + error = fwnode_property_read_u8_array(node, "prop-u8", array_u8, 17); + KUNIT_EXPECT_NE(test, error, 0); + + error = fwnode_property_read_u8(node, "no-prop-u8", &val_u8); + KUNIT_EXPECT_NE(test, error, 0); + + error = fwnode_property_read_u8_array(node, "no-prop-u8", array_u8, 1); + KUNIT_EXPECT_NE(test, error, 0); + + error = fwnode_property_read_u16(node, "prop-u16", &val_u16); + KUNIT_EXPECT_EQ(test, error, 0); + KUNIT_EXPECT_EQ(test, val_u16, 16); + + error = fwnode_property_count_u16(node, "prop-u16"); + KUNIT_EXPECT_EQ(test, error, 10); + + error = fwnode_property_read_u16_array(node, "prop-u16", array_u16, 1); + KUNIT_EXPECT_EQ(test, error, 0); + KUNIT_EXPECT_EQ(test, array_u16[0], 16); + + error = fwnode_property_read_u16_array(node, "prop-u16", array_u16, 2); + KUNIT_EXPECT_EQ(test, error, 0); + KUNIT_EXPECT_EQ(test, array_u16[0], 16); + KUNIT_EXPECT_EQ(test, array_u16[1], 17); + + error = fwnode_property_read_u16_array(node, "prop-u16", array_u16, 17); + KUNIT_EXPECT_NE(test, error, 0); + + error = fwnode_property_read_u16(node, "no-prop-u16", &val_u16); + KUNIT_EXPECT_NE(test, error, 0); + + error = fwnode_property_read_u16_array(node, "no-prop-u16", array_u16, 1); + KUNIT_EXPECT_NE(test, error, 0); + + error = fwnode_property_read_u32(node, "prop-u32", &val_u32); + KUNIT_EXPECT_EQ(test, error, 0); + KUNIT_EXPECT_EQ(test, val_u32, 32); + + error = fwnode_property_count_u32(node, "prop-u32"); + KUNIT_EXPECT_EQ(test, error, 10); + + error = fwnode_property_read_u32_array(node, "prop-u32", array_u32, 1); + KUNIT_EXPECT_EQ(test, error, 0); + KUNIT_EXPECT_EQ(test, array_u32[0], 32); + + error = fwnode_property_read_u32_array(node, "prop-u32", array_u32, 2); + KUNIT_EXPECT_EQ(test, error, 0); + KUNIT_EXPECT_EQ(test, array_u32[0], 32); + KUNIT_EXPECT_EQ(test, array_u32[1], 33); + + error = fwnode_property_read_u32_array(node, "prop-u32", array_u32, 17); + KUNIT_EXPECT_NE(test, error, 0); + + error = fwnode_property_read_u32(node, "no-prop-u32", &val_u32); + KUNIT_EXPECT_NE(test, error, 0); + + error = fwnode_property_read_u32_array(node, "no-prop-u32", array_u32, 1); + KUNIT_EXPECT_NE(test, error, 0); + + error = fwnode_property_read_u64(node, "prop-u64", &val_u64); + KUNIT_EXPECT_EQ(test, error, 0); + KUNIT_EXPECT_EQ(test, val_u64, 64); + + error = fwnode_property_count_u64(node, "prop-u64"); + KUNIT_EXPECT_EQ(test, error, 10); + + error = fwnode_property_read_u64_array(node, "prop-u64", array_u64, 1); + KUNIT_EXPECT_EQ(test, error, 0); + KUNIT_EXPECT_EQ(test, array_u64[0], 64); + + error = fwnode_property_read_u64_array(node, "prop-u64", array_u64, 2); + KUNIT_EXPECT_EQ(test, error, 0); + KUNIT_EXPECT_EQ(test, array_u64[0], 64); + KUNIT_EXPECT_EQ(test, array_u64[1], 65); + + error = fwnode_property_read_u64_array(node, "prop-u64", array_u64, 17); + KUNIT_EXPECT_NE(test, error, 0); + + error = fwnode_property_read_u64(node, "no-prop-u64", &val_u64); + KUNIT_EXPECT_NE(test, error, 0); + + error = fwnode_property_read_u64_array(node, "no-prop-u64", array_u64, 1); + KUNIT_EXPECT_NE(test, error, 0); + + /* Count 64-bit values as 16-bit */ + error = fwnode_property_count_u16(node, "prop-u64"); + KUNIT_EXPECT_EQ(test, error, 40); + + /* Other way around */ + error = fwnode_property_count_u64(node, "prop-u16"); + KUNIT_EXPECT_EQ(test, error, 2); + + fwnode_remove_software_node(node); +} + +static void pe_test_strings(struct kunit *test) +{ + static const char *strings[] = { + "string-a", + "string-b", + }; + + static const struct property_entry entries[] = { + PROPERTY_ENTRY_STRING("str", "single"), + PROPERTY_ENTRY_STRING("empty", ""), + PROPERTY_ENTRY_STRING_ARRAY("strs", strings), + { } + }; + + struct fwnode_handle *node; + const char *str; + const char *strs[10]; + int error; + + node = fwnode_create_software_node(entries, NULL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, node); + + error = fwnode_property_read_string(node, "str", &str); + KUNIT_EXPECT_EQ(test, error, 0); + KUNIT_EXPECT_STREQ(test, str, "single"); + + error = fwnode_property_string_array_count(node, "str"); + KUNIT_EXPECT_EQ(test, error, 1); + + error = fwnode_property_read_string_array(node, "str", strs, 1); + KUNIT_EXPECT_EQ(test, error, 1); + KUNIT_EXPECT_STREQ(test, strs[0], "single"); + + /* asking for more data returns what we have */ + error = fwnode_property_read_string_array(node, "str", strs, 2); + KUNIT_EXPECT_EQ(test, error, 1); + KUNIT_EXPECT_STREQ(test, strs[0], "single"); + + error = fwnode_property_read_string(node, "no-str", &str); + KUNIT_EXPECT_NE(test, error, 0); + + error = fwnode_property_read_string_array(node, "no-str", strs, 1); + KUNIT_EXPECT_LT(test, error, 0); + + error = fwnode_property_read_string(node, "empty", &str); + KUNIT_EXPECT_EQ(test, error, 0); + KUNIT_EXPECT_STREQ(test, str, ""); + + error = fwnode_property_string_array_count(node, "strs"); + KUNIT_EXPECT_EQ(test, error, 2); + + error = fwnode_property_read_string_array(node, "strs", strs, 3); + KUNIT_EXPECT_EQ(test, error, 2); + KUNIT_EXPECT_STREQ(test, strs[0], "string-a"); + KUNIT_EXPECT_STREQ(test, strs[1], "string-b"); + + error = fwnode_property_read_string_array(node, "strs", strs, 1); + KUNIT_EXPECT_EQ(test, error, 1); + KUNIT_EXPECT_STREQ(test, strs[0], "string-a"); + + /* NULL argument -> returns size */ + error = fwnode_property_read_string_array(node, "strs", NULL, 0); + KUNIT_EXPECT_EQ(test, error, 2); + + /* accessing array as single value */ + error = fwnode_property_read_string(node, "strs", &str); + KUNIT_EXPECT_EQ(test, error, 0); + KUNIT_EXPECT_STREQ(test, str, "string-a"); + + fwnode_remove_software_node(node); +} + +static void pe_test_bool(struct kunit *test) +{ + static const struct property_entry entries[] = { + PROPERTY_ENTRY_BOOL("prop"), + { } + }; + + struct fwnode_handle *node; + + node = fwnode_create_software_node(entries, NULL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, node); + + KUNIT_EXPECT_TRUE(test, fwnode_property_read_bool(node, "prop")); + KUNIT_EXPECT_FALSE(test, fwnode_property_read_bool(node, "not-prop")); + + fwnode_remove_software_node(node); +} + +/* Verifies that small U8 array is stored inline when property is copied */ +static void pe_test_move_inline_u8(struct kunit *test) +{ + static const u8 u8_array_small[8] = { 1, 2, 3, 4 }; + static const u8 u8_array_big[128] = { 5, 6, 7, 8 }; + static const struct property_entry entries[] = { + PROPERTY_ENTRY_U8_ARRAY("small", u8_array_small), + PROPERTY_ENTRY_U8_ARRAY("big", u8_array_big), + { } + }; + + struct property_entry *copy; + const u8 *data_ptr; + + copy = property_entries_dup(entries); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, copy); + + KUNIT_EXPECT_TRUE(test, copy[0].is_inline); + data_ptr = (u8 *)©[0].value; + KUNIT_EXPECT_EQ(test, data_ptr[0], 1); + KUNIT_EXPECT_EQ(test, data_ptr[1], 2); + + KUNIT_EXPECT_FALSE(test, copy[1].is_inline); + data_ptr = copy[1].pointer; + KUNIT_EXPECT_EQ(test, data_ptr[0], 5); + KUNIT_EXPECT_EQ(test, data_ptr[1], 6); + + property_entries_free(copy); +} + +/* Verifies that single string array is stored inline when property is copied */ +static void pe_test_move_inline_str(struct kunit *test) +{ + static char *str_array_small[] = { "a" }; + static char *str_array_big[] = { "b", "c", "d", "e" }; + static char *str_array_small_empty[] = { "" }; + static struct property_entry entries[] = { + PROPERTY_ENTRY_STRING_ARRAY("small", str_array_small), + PROPERTY_ENTRY_STRING_ARRAY("big", str_array_big), + PROPERTY_ENTRY_STRING_ARRAY("small-empty", str_array_small_empty), + { } + }; + + struct property_entry *copy; + const char * const *data_ptr; + + copy = property_entries_dup(entries); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, copy); + + KUNIT_EXPECT_TRUE(test, copy[0].is_inline); + KUNIT_EXPECT_STREQ(test, copy[0].value.str[0], "a"); + + KUNIT_EXPECT_FALSE(test, copy[1].is_inline); + data_ptr = copy[1].pointer; + KUNIT_EXPECT_STREQ(test, data_ptr[0], "b"); + KUNIT_EXPECT_STREQ(test, data_ptr[1], "c"); + + KUNIT_EXPECT_TRUE(test, copy[2].is_inline); + KUNIT_EXPECT_STREQ(test, copy[2].value.str[0], ""); + + property_entries_free(copy); +} + +/* Handling of reference properties */ +static void pe_test_reference(struct kunit *test) +{ + static const struct software_node node1 = { .name = "1" }; + static const struct software_node node2 = { .name = "2" }; + static const struct software_node *group[] = { &node1, &node2, NULL }; + + static const struct software_node_ref_args refs[] = { + SOFTWARE_NODE_REFERENCE(&node1), + SOFTWARE_NODE_REFERENCE(&node2, 3, 4), + }; + + const struct property_entry entries[] = { + PROPERTY_ENTRY_REF("ref-1", &node1), + PROPERTY_ENTRY_REF("ref-2", &node2, 1, 2), + PROPERTY_ENTRY_REF_ARRAY("ref-3", refs), + { } + }; + + struct fwnode_handle *node; + struct fwnode_reference_args ref; + int error; + + error = software_node_register_node_group(group); + KUNIT_ASSERT_EQ(test, error, 0); + + node = fwnode_create_software_node(entries, NULL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, node); + + error = fwnode_property_get_reference_args(node, "ref-1", NULL, + 0, 0, &ref); + KUNIT_ASSERT_EQ(test, error, 0); + KUNIT_EXPECT_PTR_EQ(test, to_software_node(ref.fwnode), &node1); + KUNIT_EXPECT_EQ(test, ref.nargs, 0U); + + /* wrong index */ + error = fwnode_property_get_reference_args(node, "ref-1", NULL, + 0, 1, &ref); + KUNIT_EXPECT_NE(test, error, 0); + + error = fwnode_property_get_reference_args(node, "ref-2", NULL, + 1, 0, &ref); + KUNIT_ASSERT_EQ(test, error, 0); + KUNIT_EXPECT_PTR_EQ(test, to_software_node(ref.fwnode), &node2); + KUNIT_EXPECT_EQ(test, ref.nargs, 1U); + KUNIT_EXPECT_EQ(test, ref.args[0], 1LLU); + + /* asking for more args, padded with zero data */ + error = fwnode_property_get_reference_args(node, "ref-2", NULL, + 3, 0, &ref); + KUNIT_ASSERT_EQ(test, error, 0); + KUNIT_EXPECT_PTR_EQ(test, to_software_node(ref.fwnode), &node2); + KUNIT_EXPECT_EQ(test, ref.nargs, 3U); + KUNIT_EXPECT_EQ(test, ref.args[0], 1LLU); + KUNIT_EXPECT_EQ(test, ref.args[1], 2LLU); + KUNIT_EXPECT_EQ(test, ref.args[2], 0LLU); + + /* wrong index */ + error = fwnode_property_get_reference_args(node, "ref-2", NULL, + 2, 1, &ref); + KUNIT_EXPECT_NE(test, error, 0); + + /* array of references */ + error = fwnode_property_get_reference_args(node, "ref-3", NULL, + 0, 0, &ref); + KUNIT_ASSERT_EQ(test, error, 0); + KUNIT_EXPECT_PTR_EQ(test, to_software_node(ref.fwnode), &node1); + KUNIT_EXPECT_EQ(test, ref.nargs, 0U); + + /* second reference in the array */ + error = fwnode_property_get_reference_args(node, "ref-3", NULL, + 2, 1, &ref); + KUNIT_ASSERT_EQ(test, error, 0); + KUNIT_EXPECT_PTR_EQ(test, to_software_node(ref.fwnode), &node2); + KUNIT_EXPECT_EQ(test, ref.nargs, 2U); + KUNIT_EXPECT_EQ(test, ref.args[0], 3LLU); + KUNIT_EXPECT_EQ(test, ref.args[1], 4LLU); + + /* wrong index */ + error = fwnode_property_get_reference_args(node, "ref-1", NULL, + 0, 2, &ref); + KUNIT_EXPECT_NE(test, error, 0); + + fwnode_remove_software_node(node); + software_node_unregister_node_group(group); +} + +static struct kunit_case property_entry_test_cases[] = { + KUNIT_CASE(pe_test_uints), + KUNIT_CASE(pe_test_uint_arrays), + KUNIT_CASE(pe_test_strings), + KUNIT_CASE(pe_test_bool), + KUNIT_CASE(pe_test_move_inline_u8), + KUNIT_CASE(pe_test_move_inline_str), + KUNIT_CASE(pe_test_reference), + { } +}; + +static struct kunit_suite property_entry_test_suite = { + .name = "property-entry", + .test_cases = property_entry_test_cases, +}; + +kunit_test_suite(property_entry_test_suite); + +MODULE_DESCRIPTION("Test module for the property entry API"); +MODULE_AUTHOR("Dmitry Torokhov <dtor@chromium.org>"); +MODULE_LICENSE("GPL"); diff --git a/drivers/base/test/root-device-test.c b/drivers/base/test/root-device-test.c new file mode 100644 index 000000000000..9aea23c9123e --- /dev/null +++ b/drivers/base/test/root-device-test.c @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright 2023 Maxime Ripard <mripard@kernel.org> + +#include <kunit/resource.h> + +#include <linux/device.h> + +#define DEVICE_NAME "test" + +struct test_priv { + bool probe_done; + bool release_done; + wait_queue_head_t release_wq; + struct device *dev; +}; + +static int root_device_devm_init(struct kunit *test) +{ + struct test_priv *priv; + + priv = kunit_kzalloc(test, sizeof(*priv), GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, priv); + init_waitqueue_head(&priv->release_wq); + + test->priv = priv; + + return 0; +} + +static void devm_device_action(void *ptr) +{ + struct test_priv *priv = ptr; + + priv->release_done = true; + wake_up_interruptible(&priv->release_wq); +} + +#define RELEASE_TIMEOUT_MS 100 + +/* + * Tests that a bus-less, non-probed device will run its device-managed + * actions when unregistered. + */ +static void root_device_devm_register_unregister_test(struct kunit *test) +{ + struct test_priv *priv = test->priv; + int ret; + + priv->dev = root_device_register(DEVICE_NAME); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, priv->dev); + + ret = devm_add_action_or_reset(priv->dev, devm_device_action, priv); + KUNIT_ASSERT_EQ(test, ret, 0); + + root_device_unregister(priv->dev); + + ret = wait_event_interruptible_timeout(priv->release_wq, priv->release_done, + msecs_to_jiffies(RELEASE_TIMEOUT_MS)); + KUNIT_EXPECT_GT(test, ret, 0); +} + +static void devm_put_device_action(void *ptr) +{ + struct test_priv *priv = ptr; + + put_device(priv->dev); + priv->release_done = true; + wake_up_interruptible(&priv->release_wq); +} + +/* + * Tests that a bus-less, non-probed device will run its device-managed + * actions when unregistered, even if someone still holds a reference to + * it. + */ +static void root_device_devm_register_get_unregister_with_devm_test(struct kunit *test) +{ + struct test_priv *priv = test->priv; + int ret; + + priv->dev = root_device_register(DEVICE_NAME); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, priv->dev); + + get_device(priv->dev); + + ret = devm_add_action_or_reset(priv->dev, devm_put_device_action, priv); + KUNIT_ASSERT_EQ(test, ret, 0); + + root_device_unregister(priv->dev); + + ret = wait_event_interruptible_timeout(priv->release_wq, priv->release_done, + msecs_to_jiffies(RELEASE_TIMEOUT_MS)); + KUNIT_EXPECT_GT(test, ret, 0); +} + +static struct kunit_case root_device_devm_tests[] = { + KUNIT_CASE(root_device_devm_register_unregister_test), + KUNIT_CASE(root_device_devm_register_get_unregister_with_devm_test), + {} +}; + +static struct kunit_suite root_device_devm_test_suite = { + .name = "root-device-devm", + .init = root_device_devm_init, + .test_cases = root_device_devm_tests, +}; + +kunit_test_suite(root_device_devm_test_suite); + +MODULE_DESCRIPTION("Test module for root devices"); +MODULE_AUTHOR("Maxime Ripard <mripard@kernel.org>"); +MODULE_LICENSE("GPL"); diff --git a/drivers/base/test/test_async_driver_probe.c b/drivers/base/test/test_async_driver_probe.c new file mode 100644 index 000000000000..3465800baa6c --- /dev/null +++ b/drivers/base/test/test_async_driver_probe.c @@ -0,0 +1,299 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2014 Google, Inc. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/delay.h> +#include <linux/init.h> +#include <linux/hrtimer.h> +#include <linux/module.h> +#include <linux/platform_device.h> +#include <linux/time.h> +#include <linux/numa.h> +#include <linux/nodemask.h> +#include <linux/topology.h> + +#define TEST_PROBE_DELAY (5 * 1000) /* 5 sec */ +#define TEST_PROBE_THRESHOLD (TEST_PROBE_DELAY / 2) + +static atomic_t warnings, errors, timeout, async_completed; + +static int test_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + + /* + * Determine if we have hit the "timeout" limit for the test if we + * have then report it as an error, otherwise we wil sleep for the + * required amount of time and then report completion. + */ + if (atomic_read(&timeout)) { + dev_err(dev, "async probe took too long\n"); + atomic_inc(&errors); + } else { + dev_dbg(&pdev->dev, "sleeping for %d msecs in probe\n", + TEST_PROBE_DELAY); + msleep(TEST_PROBE_DELAY); + dev_dbg(&pdev->dev, "done sleeping\n"); + } + + /* + * Report NUMA mismatch if device node is set and we are not + * performing an async init on that node. + */ + if (dev->driver->probe_type == PROBE_PREFER_ASYNCHRONOUS) { + if (IS_ENABLED(CONFIG_NUMA) && + dev_to_node(dev) != numa_node_id()) { + dev_warn(dev, "NUMA node mismatch %d != %d\n", + dev_to_node(dev), numa_node_id()); + atomic_inc(&warnings); + } + + atomic_inc(&async_completed); + } + + return 0; +} + +static struct platform_driver async_driver = { + .driver = { + .name = "test_async_driver", + .probe_type = PROBE_PREFER_ASYNCHRONOUS, + }, + .probe = test_probe, +}; + +static struct platform_driver sync_driver = { + .driver = { + .name = "test_sync_driver", + .probe_type = PROBE_FORCE_SYNCHRONOUS, + }, + .probe = test_probe, +}; + +static struct platform_device *async_dev[NR_CPUS * 2]; +static struct platform_device *sync_dev[2]; + +static struct platform_device * +test_platform_device_register_node(char *name, int id, int nid) +{ + struct platform_device *pdev; + int ret; + + pdev = platform_device_alloc(name, id); + if (!pdev) + return ERR_PTR(-ENOMEM); + + if (nid != NUMA_NO_NODE) + set_dev_node(&pdev->dev, nid); + + ret = platform_device_add(pdev); + if (ret) { + platform_device_put(pdev); + return ERR_PTR(ret); + } + + return pdev; + +} + +static int __init test_async_probe_init(void) +{ + struct platform_device **pdev = NULL; + int async_id = 0, sync_id = 0; + unsigned long long duration; + ktime_t calltime; + int err, nid, cpu; + + pr_info("registering first set of asynchronous devices...\n"); + + for_each_online_cpu(cpu) { + nid = cpu_to_node(cpu); + pdev = &async_dev[async_id]; + *pdev = test_platform_device_register_node("test_async_driver", + async_id, + nid); + if (IS_ERR(*pdev)) { + err = PTR_ERR(*pdev); + *pdev = NULL; + pr_err("failed to create async_dev: %d\n", err); + goto err_unregister_async_devs; + } + + async_id++; + } + + pr_info("registering asynchronous driver...\n"); + calltime = ktime_get(); + err = platform_driver_register(&async_driver); + if (err) { + pr_err("Failed to register async_driver: %d\n", err); + goto err_unregister_async_devs; + } + + duration = (unsigned long long)ktime_ms_delta(ktime_get(), calltime); + pr_info("registration took %lld msecs\n", duration); + if (duration > TEST_PROBE_THRESHOLD) { + pr_err("test failed: probe took too long\n"); + err = -ETIMEDOUT; + goto err_unregister_async_driver; + } + + pr_info("registering second set of asynchronous devices...\n"); + calltime = ktime_get(); + for_each_online_cpu(cpu) { + nid = cpu_to_node(cpu); + pdev = &async_dev[async_id]; + + *pdev = test_platform_device_register_node("test_async_driver", + async_id, + nid); + if (IS_ERR(*pdev)) { + err = PTR_ERR(*pdev); + *pdev = NULL; + pr_err("failed to create async_dev: %d\n", err); + goto err_unregister_async_driver; + } + + async_id++; + } + + duration = (unsigned long long)ktime_ms_delta(ktime_get(), calltime); + dev_info(&(*pdev)->dev, + "registration took %lld msecs\n", duration); + if (duration > TEST_PROBE_THRESHOLD) { + dev_err(&(*pdev)->dev, + "test failed: probe took too long\n"); + err = -ETIMEDOUT; + goto err_unregister_async_driver; + } + + + pr_info("registering first synchronous device...\n"); + nid = cpu_to_node(cpu); + pdev = &sync_dev[sync_id]; + + *pdev = test_platform_device_register_node("test_sync_driver", + sync_id, + NUMA_NO_NODE); + if (IS_ERR(*pdev)) { + err = PTR_ERR(*pdev); + *pdev = NULL; + pr_err("failed to create sync_dev: %d\n", err); + goto err_unregister_async_driver; + } + + sync_id++; + + pr_info("registering synchronous driver...\n"); + calltime = ktime_get(); + err = platform_driver_register(&sync_driver); + if (err) { + pr_err("Failed to register async_driver: %d\n", err); + goto err_unregister_sync_devs; + } + + duration = (unsigned long long)ktime_ms_delta(ktime_get(), calltime); + pr_info("registration took %lld msecs\n", duration); + if (duration < TEST_PROBE_THRESHOLD) { + dev_err(&(*pdev)->dev, + "test failed: probe was too quick\n"); + err = -ETIMEDOUT; + goto err_unregister_sync_driver; + } + + pr_info("registering second synchronous device...\n"); + pdev = &sync_dev[sync_id]; + calltime = ktime_get(); + + *pdev = test_platform_device_register_node("test_sync_driver", + sync_id, + NUMA_NO_NODE); + if (IS_ERR(*pdev)) { + err = PTR_ERR(*pdev); + *pdev = NULL; + pr_err("failed to create sync_dev: %d\n", err); + goto err_unregister_sync_driver; + } + + sync_id++; + + duration = (unsigned long long)ktime_ms_delta(ktime_get(), calltime); + dev_info(&(*pdev)->dev, + "registration took %lld msecs\n", duration); + if (duration < TEST_PROBE_THRESHOLD) { + dev_err(&(*pdev)->dev, + "test failed: probe was too quick\n"); + err = -ETIMEDOUT; + goto err_unregister_sync_driver; + } + + /* + * The async events should have completed while we were taking care + * of the synchronous events. We will now terminate any outstanding + * asynchronous probe calls remaining by forcing timeout and remove + * the driver before we return which should force the flush of the + * pending asynchronous probe calls. + * + * Otherwise if they completed without errors or warnings then + * report successful completion. + */ + if (atomic_read(&async_completed) != async_id) { + pr_err("async events still pending, forcing timeout\n"); + atomic_inc(&timeout); + err = -ETIMEDOUT; + } else if (!atomic_read(&errors) && !atomic_read(&warnings)) { + pr_info("completed successfully\n"); + return 0; + } + +err_unregister_sync_driver: + platform_driver_unregister(&sync_driver); +err_unregister_sync_devs: + while (sync_id--) + platform_device_unregister(sync_dev[sync_id]); +err_unregister_async_driver: + platform_driver_unregister(&async_driver); +err_unregister_async_devs: + while (async_id--) + platform_device_unregister(async_dev[async_id]); + + /* + * If err is already set then count that as an additional error for + * the test. Otherwise we will report an invalid argument error and + * not count that as we should have reached here as a result of + * errors or warnings being reported by the probe routine. + */ + if (err) + atomic_inc(&errors); + else + err = -EINVAL; + + pr_err("Test failed with %d errors and %d warnings\n", + atomic_read(&errors), atomic_read(&warnings)); + + return err; +} +module_init(test_async_probe_init); + +static void __exit test_async_probe_exit(void) +{ + int id = 2; + + platform_driver_unregister(&async_driver); + platform_driver_unregister(&sync_driver); + + while (id--) + platform_device_unregister(sync_dev[id]); + + id = NR_CPUS * 2; + while (id--) + platform_device_unregister(async_dev[id]); +} +module_exit(test_async_probe_exit); + +MODULE_DESCRIPTION("Test module for asynchronous driver probing"); +MODULE_AUTHOR("Dmitry Torokhov <dtor@chromium.org>"); +MODULE_LICENSE("GPL"); diff --git a/drivers/base/topology.c b/drivers/base/topology.c index ae989c57cd5e..c890e2a5b428 100644 --- a/drivers/base/topology.c +++ b/drivers/base/topology.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0+ /* * driver/base/topology.c - Populate sysfs with cpu topology information * @@ -6,191 +7,256 @@ * Copyright (C) 2006, Intel Corp. * * All rights reserved. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or - * NON INFRINGEMENT. See the GNU General Public License for more - * details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - * */ -#include <linux/init.h> #include <linux/mm.h> #include <linux/cpu.h> #include <linux/module.h> #include <linux/hardirq.h> #include <linux/topology.h> -#define define_one_ro_named(_name, _func) \ - static DEVICE_ATTR(_name, 0444, _func, NULL) - -#define define_one_ro(_name) \ - static DEVICE_ATTR(_name, 0444, show_##_name, NULL) - -#define define_id_show_func(name) \ -static ssize_t show_##name(struct device *dev, \ - struct device_attribute *attr, char *buf) \ -{ \ - unsigned int cpu = dev->id; \ - return sprintf(buf, "%d\n", topology_##name(cpu)); \ -} - -#if defined(topology_thread_cpumask) || defined(topology_core_cpumask) || \ - defined(topology_book_cpumask) -static ssize_t show_cpumap(int type, const struct cpumask *mask, char *buf) -{ - ptrdiff_t len = PTR_ALIGN(buf + PAGE_SIZE - 1, PAGE_SIZE) - buf; - int n = 0; - - if (len > 1) { - n = type? - cpulist_scnprintf(buf, len-2, mask) : - cpumask_scnprintf(buf, len-2, mask); - buf[n++] = '\n'; - buf[n] = '\0'; - } - return n; -} -#endif - -#ifdef arch_provides_topology_pointers -#define define_siblings_show_map(name) \ -static ssize_t show_##name(struct device *dev, \ +#define define_id_show_func(name, fmt) \ +static ssize_t name##_show(struct device *dev, \ struct device_attribute *attr, char *buf) \ { \ - unsigned int cpu = dev->id; \ - return show_cpumap(0, topology_##name(cpu), buf); \ + return sysfs_emit(buf, fmt "\n", topology_##name(dev->id)); \ } -#define define_siblings_show_list(name) \ -static ssize_t show_##name##_list(struct device *dev, \ - struct device_attribute *attr, \ - char *buf) \ -{ \ - unsigned int cpu = dev->id; \ - return show_cpumap(1, topology_##name(cpu), buf); \ +#define define_siblings_read_func(name, mask) \ +static ssize_t name##_read(struct file *file, struct kobject *kobj, \ + const struct bin_attribute *attr, char *buf, \ + loff_t off, size_t count) \ +{ \ + struct device *dev = kobj_to_dev(kobj); \ + cpumask_var_t mask; \ + ssize_t n; \ + \ + if (!alloc_cpumask_var(&mask, GFP_KERNEL)) \ + return -ENOMEM; \ + \ + cpumask_copy(mask, topology_##mask(dev->id)); \ + n = cpumap_print_bitmask_to_buf(buf, mask, off, count); \ + free_cpumask_var(mask); \ + \ + return n; \ +} \ + \ +static ssize_t name##_list_read(struct file *file, struct kobject *kobj, \ + const struct bin_attribute *attr, char *buf, \ + loff_t off, size_t count) \ +{ \ + struct device *dev = kobj_to_dev(kobj); \ + cpumask_var_t mask; \ + ssize_t n; \ + \ + if (!alloc_cpumask_var(&mask, GFP_KERNEL)) \ + return -ENOMEM; \ + \ + cpumask_copy(mask, topology_##mask(dev->id)); \ + n = cpumap_print_list_to_buf(buf, mask, off, count); \ + free_cpumask_var(mask); \ + \ + return n; \ } -#else -#define define_siblings_show_map(name) \ -static ssize_t show_##name(struct device *dev, \ - struct device_attribute *attr, char *buf) \ -{ \ - return show_cpumap(0, topology_##name(dev->id), buf); \ -} +define_id_show_func(physical_package_id, "%d"); +static DEVICE_ATTR_RO(physical_package_id); -#define define_siblings_show_list(name) \ -static ssize_t show_##name##_list(struct device *dev, \ - struct device_attribute *attr, \ - char *buf) \ -{ \ - return show_cpumap(1, topology_##name(dev->id), buf); \ -} +#ifdef TOPOLOGY_DIE_SYSFS +define_id_show_func(die_id, "%d"); +static DEVICE_ATTR_RO(die_id); +#endif + +#ifdef TOPOLOGY_CLUSTER_SYSFS +define_id_show_func(cluster_id, "%d"); +static DEVICE_ATTR_RO(cluster_id); #endif -#define define_siblings_show_func(name) \ - define_siblings_show_map(name); define_siblings_show_list(name) +define_id_show_func(core_id, "%d"); +static DEVICE_ATTR_RO(core_id); + +define_id_show_func(ppin, "0x%llx"); +static DEVICE_ATTR_ADMIN_RO(ppin); + +define_siblings_read_func(thread_siblings, sibling_cpumask); +static const BIN_ATTR_RO(thread_siblings, CPUMAP_FILE_MAX_BYTES); +static const BIN_ATTR_RO(thread_siblings_list, CPULIST_FILE_MAX_BYTES); + +define_siblings_read_func(core_cpus, sibling_cpumask); +static const BIN_ATTR_RO(core_cpus, CPUMAP_FILE_MAX_BYTES); +static const BIN_ATTR_RO(core_cpus_list, CPULIST_FILE_MAX_BYTES); + +define_siblings_read_func(core_siblings, core_cpumask); +static const BIN_ATTR_RO(core_siblings, CPUMAP_FILE_MAX_BYTES); +static const BIN_ATTR_RO(core_siblings_list, CPULIST_FILE_MAX_BYTES); + +#ifdef TOPOLOGY_CLUSTER_SYSFS +define_siblings_read_func(cluster_cpus, cluster_cpumask); +static const BIN_ATTR_RO(cluster_cpus, CPUMAP_FILE_MAX_BYTES); +static const BIN_ATTR_RO(cluster_cpus_list, CPULIST_FILE_MAX_BYTES); +#endif -define_id_show_func(physical_package_id); -define_one_ro(physical_package_id); +#ifdef TOPOLOGY_DIE_SYSFS +define_siblings_read_func(die_cpus, die_cpumask); +static const BIN_ATTR_RO(die_cpus, CPUMAP_FILE_MAX_BYTES); +static const BIN_ATTR_RO(die_cpus_list, CPULIST_FILE_MAX_BYTES); +#endif -define_id_show_func(core_id); -define_one_ro(core_id); +define_siblings_read_func(package_cpus, core_cpumask); +static const BIN_ATTR_RO(package_cpus, CPUMAP_FILE_MAX_BYTES); +static const BIN_ATTR_RO(package_cpus_list, CPULIST_FILE_MAX_BYTES); -define_siblings_show_func(thread_cpumask); -define_one_ro_named(thread_siblings, show_thread_cpumask); -define_one_ro_named(thread_siblings_list, show_thread_cpumask_list); +#ifdef TOPOLOGY_BOOK_SYSFS +define_id_show_func(book_id, "%d"); +static DEVICE_ATTR_RO(book_id); +define_siblings_read_func(book_siblings, book_cpumask); +static const BIN_ATTR_RO(book_siblings, CPUMAP_FILE_MAX_BYTES); +static const BIN_ATTR_RO(book_siblings_list, CPULIST_FILE_MAX_BYTES); +#endif -define_siblings_show_func(core_cpumask); -define_one_ro_named(core_siblings, show_core_cpumask); -define_one_ro_named(core_siblings_list, show_core_cpumask_list); +#ifdef TOPOLOGY_DRAWER_SYSFS +define_id_show_func(drawer_id, "%d"); +static DEVICE_ATTR_RO(drawer_id); +define_siblings_read_func(drawer_siblings, drawer_cpumask); +static const BIN_ATTR_RO(drawer_siblings, CPUMAP_FILE_MAX_BYTES); +static const BIN_ATTR_RO(drawer_siblings_list, CPULIST_FILE_MAX_BYTES); +#endif -#ifdef CONFIG_SCHED_BOOK -define_id_show_func(book_id); -define_one_ro(book_id); -define_siblings_show_func(book_cpumask); -define_one_ro_named(book_siblings, show_book_cpumask); -define_one_ro_named(book_siblings_list, show_book_cpumask_list); +static const struct bin_attribute *const bin_attrs[] = { + &bin_attr_core_cpus, + &bin_attr_core_cpus_list, + &bin_attr_thread_siblings, + &bin_attr_thread_siblings_list, + &bin_attr_core_siblings, + &bin_attr_core_siblings_list, +#ifdef TOPOLOGY_CLUSTER_SYSFS + &bin_attr_cluster_cpus, + &bin_attr_cluster_cpus_list, #endif +#ifdef TOPOLOGY_DIE_SYSFS + &bin_attr_die_cpus, + &bin_attr_die_cpus_list, +#endif + &bin_attr_package_cpus, + &bin_attr_package_cpus_list, +#ifdef TOPOLOGY_BOOK_SYSFS + &bin_attr_book_siblings, + &bin_attr_book_siblings_list, +#endif +#ifdef TOPOLOGY_DRAWER_SYSFS + &bin_attr_drawer_siblings, + &bin_attr_drawer_siblings_list, +#endif + NULL +}; static struct attribute *default_attrs[] = { &dev_attr_physical_package_id.attr, +#ifdef TOPOLOGY_DIE_SYSFS + &dev_attr_die_id.attr, +#endif +#ifdef TOPOLOGY_CLUSTER_SYSFS + &dev_attr_cluster_id.attr, +#endif &dev_attr_core_id.attr, - &dev_attr_thread_siblings.attr, - &dev_attr_thread_siblings_list.attr, - &dev_attr_core_siblings.attr, - &dev_attr_core_siblings_list.attr, -#ifdef CONFIG_SCHED_BOOK +#ifdef TOPOLOGY_BOOK_SYSFS &dev_attr_book_id.attr, - &dev_attr_book_siblings.attr, - &dev_attr_book_siblings_list.attr, #endif +#ifdef TOPOLOGY_DRAWER_SYSFS + &dev_attr_drawer_id.attr, +#endif + &dev_attr_ppin.attr, NULL }; -static struct attribute_group topology_attr_group = { +static umode_t topology_is_visible(struct kobject *kobj, + struct attribute *attr, int unused) +{ + if (attr == &dev_attr_ppin.attr && !topology_ppin(kobj_to_dev(kobj)->id)) + return 0; + + return attr->mode; +} + +static const struct attribute_group topology_attr_group = { .attrs = default_attrs, + .bin_attrs = bin_attrs, + .is_visible = topology_is_visible, .name = "topology" }; /* Add/Remove cpu_topology interface for CPU device */ -static int __cpuinit topology_add_dev(unsigned int cpu) +static int topology_add_dev(unsigned int cpu) { struct device *dev = get_cpu_device(cpu); return sysfs_create_group(&dev->kobj, &topology_attr_group); } -static void __cpuinit topology_remove_dev(unsigned int cpu) +static int topology_remove_dev(unsigned int cpu) { struct device *dev = get_cpu_device(cpu); sysfs_remove_group(&dev->kobj, &topology_attr_group); + return 0; } -static int __cpuinit topology_cpu_callback(struct notifier_block *nfb, - unsigned long action, void *hcpu) +static int __init topology_sysfs_init(void) { - unsigned int cpu = (unsigned long)hcpu; - int rc = 0; - - switch (action) { - case CPU_UP_PREPARE: - case CPU_UP_PREPARE_FROZEN: - rc = topology_add_dev(cpu); - break; - case CPU_UP_CANCELED: - case CPU_UP_CANCELED_FROZEN: - case CPU_DEAD: - case CPU_DEAD_FROZEN: - topology_remove_dev(cpu); - break; - } - return notifier_from_errno(rc); + return cpuhp_setup_state(CPUHP_TOPOLOGY_PREPARE, + "base/topology:prepare", topology_add_dev, + topology_remove_dev); } -static int __cpuinit topology_sysfs_init(void) +device_initcall(topology_sysfs_init); + +DEFINE_PER_CPU(unsigned long, cpu_scale) = SCHED_CAPACITY_SCALE; +EXPORT_PER_CPU_SYMBOL_GPL(cpu_scale); + +void topology_set_cpu_scale(unsigned int cpu, unsigned long capacity) +{ + per_cpu(cpu_scale, cpu) = capacity; +} + +static ssize_t cpu_capacity_show(struct device *dev, + struct device_attribute *attr, + char *buf) { - int cpu; - int rc; + struct cpu *cpu = container_of(dev, struct cpu, dev); + + return sysfs_emit(buf, "%lu\n", topology_get_cpu_scale(cpu->dev.id)); +} - for_each_online_cpu(cpu) { - rc = topology_add_dev(cpu); - if (rc) - return rc; - } - hotcpu_notifier(topology_cpu_callback, 0); +static DEVICE_ATTR_RO(cpu_capacity); + +static int cpu_capacity_sysctl_add(unsigned int cpu) +{ + struct device *cpu_dev = get_cpu_device(cpu); + + if (!cpu_dev) + return -ENOENT; + + device_create_file(cpu_dev, &dev_attr_cpu_capacity); return 0; } -device_initcall(topology_sysfs_init); +static int cpu_capacity_sysctl_remove(unsigned int cpu) +{ + struct device *cpu_dev = get_cpu_device(cpu); + + if (!cpu_dev) + return -ENOENT; + + device_remove_file(cpu_dev, &dev_attr_cpu_capacity); + + return 0; +} + +static int register_cpu_capacity_sysctl(void) +{ + cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "topology/cpu-capacity", + cpu_capacity_sysctl_add, cpu_capacity_sysctl_remove); + + return 0; +} +subsys_initcall(register_cpu_capacity_sysctl); diff --git a/drivers/base/trace.c b/drivers/base/trace.c new file mode 100644 index 000000000000..b24b0a309c4a --- /dev/null +++ b/drivers/base/trace.c @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Device core Trace Support + * Copyright (C) 2021, Intel Corporation + * + * Author: Andy Shevchenko <andriy.shevchenko@linux.intel.com> + */ + +#define CREATE_TRACE_POINTS +#include "trace.h" diff --git a/drivers/base/trace.h b/drivers/base/trace.h new file mode 100644 index 000000000000..3b83b13a57ff --- /dev/null +++ b/drivers/base/trace.h @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Device core Trace Support + * Copyright (C) 2021, Intel Corporation + * + * Author: Andy Shevchenko <andriy.shevchenko@linux.intel.com> + */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM dev + +#if !defined(__DEV_TRACE_H) || defined(TRACE_HEADER_MULTI_READ) +#define __DEV_TRACE_H + +#include <linux/device.h> +#include <linux/tracepoint.h> +#include <linux/types.h> + +DECLARE_EVENT_CLASS(devres, + TP_PROTO(struct device *dev, const char *op, void *node, const char *name, size_t size), + TP_ARGS(dev, op, node, name, size), + TP_STRUCT__entry( + __string(devname, dev_name(dev)) + __field(struct device *, dev) + __field(const char *, op) + __field(void *, node) + __string(name, name) + __field(size_t, size) + ), + TP_fast_assign( + __assign_str(devname); + __entry->op = op; + __entry->node = node; + __assign_str(name); + __entry->size = size; + ), + TP_printk("%s %3s %p %s (%zu bytes)", __get_str(devname), + __entry->op, __entry->node, __get_str(name), __entry->size) +); + +DEFINE_EVENT(devres, devres_log, + TP_PROTO(struct device *dev, const char *op, void *node, const char *name, size_t size), + TP_ARGS(dev, op, node, name, size) +); + +#endif /* __DEV_TRACE_H */ + +/* this part has to be here */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . + +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE trace + +#include <trace/define_trace.h> diff --git a/drivers/base/transport_class.c b/drivers/base/transport_class.c index f6c453c3816e..09ee2a1e35bb 100644 --- a/drivers/base/transport_class.c +++ b/drivers/base/transport_class.c @@ -1,11 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0 /* * transport_class.c - implementation of generic transport classes * using attribute_containers * * Copyright (c) 2005 - James Bottomley <James.Bottomley@steeleye.com> * - * This file is licensed under GPLv2 - * * The basic idea here is to allow any "device controller" (which * would most often be a Host Bus Adapter to use the services of one * or more tranport classes for performing transport specific @@ -31,6 +30,10 @@ #include <linux/attribute_container.h> #include <linux/transport_class.h> +static int transport_remove_classdev(struct attribute_container *cont, + struct device *dev, + struct device *classdev); + /** * transport_class_register - register an initial transport class * @@ -152,12 +155,27 @@ static int transport_add_class_device(struct attribute_container *cont, struct device *dev, struct device *classdev) { + struct transport_class *tclass = class_to_transport_class(cont->class); int error = attribute_container_add_class_device(classdev); struct transport_container *tcont = attribute_container_to_transport_container(cont); - if (!error && tcont->statistics) + if (error) + goto err_remove; + + if (tcont->statistics) { error = sysfs_create_group(&classdev->kobj, tcont->statistics); + if (error) + goto err_del; + } + + return 0; + +err_del: + attribute_container_class_device_del(classdev); +err_remove: + if (tclass->remove) + tclass->remove(tcont, dev, classdev); return error; } @@ -173,10 +191,11 @@ static int transport_add_class_device(struct attribute_container *cont, * routine is simply a trigger point used to add the device to the * system and register attributes for it. */ - -void transport_add_device(struct device *dev) +int transport_add_device(struct device *dev) { - attribute_container_device_trigger(dev, transport_add_class_device); + return attribute_container_device_trigger_safe(dev, + transport_add_class_device, + transport_remove_classdev); } EXPORT_SYMBOL_GPL(transport_add_device); |