diff options
Diffstat (limited to 'drivers/cpufreq')
130 files changed, 26392 insertions, 15353 deletions
diff --git a/drivers/cpufreq/Kconfig b/drivers/cpufreq/Kconfig index 534fcb825153..78702a08364f 100644 --- a/drivers/cpufreq/Kconfig +++ b/drivers/cpufreq/Kconfig @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only menu "CPU Frequency scaling" config CPU_FREQ @@ -11,47 +12,37 @@ config CPU_FREQ clock speed, you need to either enable a dynamic cpufreq governor (see below) after boot, or use a userspace tool. - For details, take a look at <file:Documentation/cpu-freq>. + For details, take a look at + <file:Documentation/admin-guide/pm/cpufreq.rst>. If in doubt, say N. if CPU_FREQ -config CPU_FREQ_TABLE - tristate +config CPU_FREQ_GOV_ATTR_SET + bool config CPU_FREQ_GOV_COMMON + select CPU_FREQ_GOV_ATTR_SET + select IRQ_WORK bool config CPU_FREQ_STAT - tristate "CPU frequency translation statistics" - select CPU_FREQ_TABLE - default y + bool "CPU frequency transition statistics" help - This driver exports CPU frequency statistics information through sysfs - file system. - - To compile this driver as a module, choose M here: the - module will be called cpufreq_stats. - - If in doubt, say N. - -config CPU_FREQ_STAT_DETAILS - bool "CPU frequency translation statistics details" - depends on CPU_FREQ_STAT - help - This will show detail CPU frequency translation table in sysfs file - system. + Export CPU frequency statistics information through sysfs. If in doubt, say N. choice prompt "Default CPUFreq governor" - default CPU_FREQ_DEFAULT_GOV_USERSPACE if ARM_SA1100_CPUFREQ || ARM_SA1110_CPUFREQ + default CPU_FREQ_DEFAULT_GOV_USERSPACE if ARM_SA1110_CPUFREQ + default CPU_FREQ_DEFAULT_GOV_SCHEDUTIL if ARM64 || ARM + default CPU_FREQ_DEFAULT_GOV_SCHEDUTIL if (X86_INTEL_PSTATE || X86_AMD_PSTATE) && SMP default CPU_FREQ_DEFAULT_GOV_PERFORMANCE help This option sets which CPUFreq governor shall be loaded at - startup. If in doubt, select 'performance'. + startup. If in doubt, use the default setting. config CPU_FREQ_DEFAULT_GOV_PERFORMANCE bool "performance" @@ -63,7 +54,6 @@ config CPU_FREQ_DEFAULT_GOV_PERFORMANCE config CPU_FREQ_DEFAULT_GOV_POWERSAVE bool "powersave" - depends on EXPERT select CPU_FREQ_GOV_POWERSAVE help Use the CPUFreq governor 'powersave' as default. This sets @@ -81,6 +71,7 @@ config CPU_FREQ_DEFAULT_GOV_USERSPACE config CPU_FREQ_DEFAULT_GOV_ONDEMAND bool "ondemand" + depends on !(X86_INTEL_PSTATE && SMP) select CPU_FREQ_GOV_ONDEMAND select CPU_FREQ_GOV_PERFORMANCE help @@ -93,6 +84,7 @@ config CPU_FREQ_DEFAULT_GOV_ONDEMAND config CPU_FREQ_DEFAULT_GOV_CONSERVATIVE bool "conservative" + depends on !(X86_INTEL_PSTATE && SMP) select CPU_FREQ_GOV_CONSERVATIVE select CPU_FREQ_GOV_PERFORMANCE help @@ -102,6 +94,17 @@ config CPU_FREQ_DEFAULT_GOV_CONSERVATIVE Be aware that not all cpufreq drivers support the conservative governor. If unsure have a look at the help section of the driver. Fallback governor will be the performance governor. + +config CPU_FREQ_DEFAULT_GOV_SCHEDUTIL + bool "schedutil" + depends on SMP + select CPU_FREQ_GOV_SCHEDUTIL + select CPU_FREQ_GOV_PERFORMANCE + help + Use the 'schedutil' CPUFreq governor by default. If unsure, + have a look at the help section of that governor. The fallback + governor will be 'performance'. + endchoice config CPU_FREQ_GOV_PERFORMANCE @@ -137,13 +140,10 @@ config CPU_FREQ_GOV_USERSPACE To compile this driver as a module, choose M here: the module will be called cpufreq_userspace. - For details, take a look at <file:Documentation/cpu-freq/>. - If in doubt, say Y. config CPU_FREQ_GOV_ONDEMAND tristate "'ondemand' cpufreq policy governor" - select CPU_FREQ_TABLE select CPU_FREQ_GOV_COMMON help 'ondemand' - This driver adds a dynamic cpufreq policy governor. @@ -156,7 +156,8 @@ config CPU_FREQ_GOV_ONDEMAND To compile this driver as a module, choose M here: the module will be called cpufreq_ondemand. - For details, take a look at linux/Documentation/cpu-freq. + For details, take a look at + <file:Documentation/admin-guide/pm/cpufreq.rst>. If in doubt, say N. @@ -180,112 +181,142 @@ config CPU_FREQ_GOV_CONSERVATIVE To compile this driver as a module, choose M here: the module will be called cpufreq_conservative. - For details, take a look at linux/Documentation/cpu-freq. + For details, take a look at + <file:Documentation/admin-guide/pm/cpufreq.rst>. If in doubt, say N. -config GENERIC_CPUFREQ_CPU0 - tristate "Generic CPU0 cpufreq driver" - depends on HAVE_CLK && REGULATOR && PM_OPP && OF - select CPU_FREQ_TABLE +config CPU_FREQ_GOV_SCHEDUTIL + bool "'schedutil' cpufreq policy governor" + depends on CPU_FREQ && SMP + select CPU_FREQ_GOV_ATTR_SET + select IRQ_WORK help - This adds a generic cpufreq driver for CPU0 frequency management. - It supports both uniprocessor (UP) and symmetric multiprocessor (SMP) - systems which share clock and voltage across all CPUs. + This governor makes decisions based on the utilization data provided + by the scheduler. It sets the CPU frequency to be proportional to + the utilization/capacity ratio coming from the scheduler. If the + utilization is frequency-invariant, the new frequency is also + proportional to the maximum available frequency. If that is not the + case, it is proportional to the current frequency of the CPU. The + frequency tipping point is at utilization/capacity equal to 80% in + both cases. If in doubt, say N. -menu "x86 CPU frequency scaling drivers" -depends on X86 -source "drivers/cpufreq/Kconfig.x86" -endmenu +comment "CPU frequency scaling drivers" -menu "ARM CPU frequency scaling drivers" -depends on ARM -source "drivers/cpufreq/Kconfig.arm" -endmenu +config CPUFREQ_DT + tristate "Generic DT based cpufreq driver" + depends on HAVE_CLK && OF + select CPUFREQ_DT_PLATDEV + select PM_OPP + help + This adds a generic DT based cpufreq driver for frequency management. + It supports both uniprocessor (UP) and symmetric multiprocessor (SMP) + systems. -menu "AVR32 CPU frequency scaling drivers" -depends on AVR32 + If in doubt, say N. -config AVR32_AT32AP_CPUFREQ - bool "CPU frequency driver for AT32AP" - depends on PLATFORM_AT32AP - default n +config CPUFREQ_DT_RUST + tristate "Rust based Generic DT based cpufreq driver" + depends on HAVE_CLK && OF && RUST + select CPUFREQ_DT_PLATDEV + select PM_OPP help - This enables the CPU frequency driver for AT32AP processors. + This adds a Rust based generic DT based cpufreq driver for frequency + management. It supports both uniprocessor (UP) and symmetric + multiprocessor (SMP) systems. + If in doubt, say N. -endmenu +config CPUFREQ_VIRT + tristate "Virtual cpufreq driver" + depends on GENERIC_ARCH_TOPOLOGY + help + This adds a virtualized cpufreq driver for guest kernels that + read/writes to a MMIO region for a virtualized cpufreq device to + communicate with the host. It sends performance requests to the host + which gets used as a hint to schedule vCPU threads and select CPU + frequency. If a VM does not support a virtualized FIE such as AMUs, + it updates the frequency scaling factor by polling host CPU frequency + to enable accurate Per-Entity Load Tracking for tasks running in the guest. -menu "CPUFreq processor drivers" -depends on IA64 + If in doubt, say N. -config IA64_ACPI_CPUFREQ - tristate "ACPI Processor P-States driver" - select CPU_FREQ_TABLE - depends on ACPI_PROCESSOR +config CPUFREQ_DT_PLATDEV + bool "Generic DT based cpufreq platdev driver" + depends on OF help - This driver adds a CPUFreq driver which utilizes the ACPI - Processor Performance States. + This adds a generic DT based cpufreq platdev driver for frequency + management. This creates a 'cpufreq-dt' platform device, on the + supported platforms. - For details, take a look at <file:Documentation/cpu-freq/>. + If in doubt, say N. - If in doubt, say N. +if X86 +source "drivers/cpufreq/Kconfig.x86" +endif -endmenu +source "drivers/cpufreq/Kconfig.arm" + +if PPC32 || PPC64 +source "drivers/cpufreq/Kconfig.powerpc" +endif -menu "MIPS CPUFreq processor drivers" -depends on MIPS +if MIPS +config BMIPS_CPUFREQ + tristate "BMIPS CPUfreq Driver" + help + This option adds a CPUfreq driver for BMIPS processors with + support for configurable CPU frequency. + + For now, BMIPS5 chips are supported (such as the Broadcom 7425). + + If in doubt, say N. config LOONGSON2_CPUFREQ tristate "Loongson2 CPUFreq Driver" - select CPU_FREQ_TABLE + depends on LEMOTE_MACH2F help This option adds a CPUFreq driver for loongson processors which support software configurable cpu frequency. - Loongson2F and it's successors support this feature. - - For details, take a look at <file:Documentation/cpu-freq/>. + Loongson2F and its successors support this feature. If in doubt, say N. +endif -endmenu +if LOONGARCH +config LOONGSON3_CPUFREQ + tristate "Loongson3 CPUFreq Driver" + help + This option adds a CPUFreq driver for Loongson processors which + support software configurable cpu frequency. -menu "PowerPC CPU frequency scaling drivers" -depends on PPC32 || PPC64 -source "drivers/cpufreq/Kconfig.powerpc" -endmenu + Loongson-3 family processors support this feature. + + If in doubt, say N. +endif -menu "SPARC CPU frequency scaling drivers" -depends on SPARC64 +if SPARC64 config SPARC_US3_CPUFREQ tristate "UltraSPARC-III CPU Frequency driver" - select CPU_FREQ_TABLE help This adds the CPUFreq driver for UltraSPARC-III processors. - For details, take a look at <file:Documentation/cpu-freq>. - If in doubt, say N. config SPARC_US2E_CPUFREQ tristate "UltraSPARC-IIe CPU Frequency driver" - select CPU_FREQ_TABLE help This adds the CPUFreq driver for UltraSPARC-IIe processors. - For details, take a look at <file:Documentation/cpu-freq>. - If in doubt, say N. -endmenu +endif -menu "SH CPU Frequency scaling" -depends on SUPERH +if SUPERH config SH_CPU_FREQ tristate "SuperH CPU Frequency driver" - select CPU_FREQ_TABLE help This adds the cpufreq driver for SuperH. Any CPU that supports clock rate rounding through the clock framework can use this @@ -294,10 +325,46 @@ config SH_CPU_FREQ will also generate a notice in the boot log before disabling itself if the CPU in question is not capable of rate rounding. - For details, take a look at <file:Documentation/cpu-freq>. - If unsure, say N. -endmenu +endif + +config QORIQ_CPUFREQ + tristate "CPU frequency scaling driver for Freescale QorIQ SoCs" + depends on OF && COMMON_CLK + depends on PPC_E500MC || SOC_LS1021A || ARCH_LAYERSCAPE || COMPILE_TEST + select CLK_QORIQ + help + This adds the CPUFreq driver support for Freescale QorIQ SoCs + which are capable of changing the CPU's frequency dynamically. + +config ACPI_CPPC_CPUFREQ + tristate "CPUFreq driver based on the ACPI CPPC spec" + depends on ACPI_PROCESSOR + depends on ARM || ARM64 || RISCV + select ACPI_CPPC_LIB + help + This adds a CPUFreq driver which uses CPPC methods + as described in the ACPIv5.1 spec. CPPC stands for + Collaborative Processor Performance Controls. It + is based on an abstract continuous scale of CPU + performance values which allows the remote power + processor to flexibly optimize for power and + performance. CPPC relies on power management firmware + support for its operation. + + If in doubt, say N. + +config ACPI_CPPC_CPUFREQ_FIE + bool "Frequency Invariance support for CPPC cpufreq driver" + depends on ACPI_CPPC_CPUFREQ && GENERIC_ARCH_TOPOLOGY + depends on ARM || ARM64 || RISCV + default y + help + This extends frequency invariance support in the CPPC cpufreq driver, + by using CPPC delivered and reference performance counters. + + If in doubt, say N. endif + endmenu diff --git a/drivers/cpufreq/Kconfig.arm b/drivers/cpufreq/Kconfig.arm index de4d5d93c3fd..9be0503df55a 100644 --- a/drivers/cpufreq/Kconfig.arm +++ b/drivers/cpufreq/Kconfig.arm @@ -1,68 +1,93 @@ +# SPDX-License-Identifier: GPL-2.0-only # # ARM CPU Frequency scaling drivers # -config ARM_BIG_LITTLE_CPUFREQ - tristate "Generic ARM big LITTLE CPUfreq driver" - depends on ARM_CPU_TOPOLOGY && PM_OPP && HAVE_CLK - select CPU_FREQ_TABLE +config ARM_ALLWINNER_SUN50I_CPUFREQ_NVMEM + tristate "Allwinner nvmem based SUN50I CPUFreq driver" + depends on ARCH_SUNXI || COMPILE_TEST + depends on NVMEM_SUNXI_SID + select PM_OPP + help + This adds the nvmem based CPUFreq driver for Allwinner + h6 SoC. + + To compile this driver as a module, choose M here: the + module will be called sun50i-cpufreq-nvmem. + +config ARM_AIROHA_SOC_CPUFREQ + tristate "Airoha EN7581 SoC CPUFreq support" + depends on ARCH_AIROHA || COMPILE_TEST + depends on OF + select PM_OPP + default ARCH_AIROHA help - This enables the Generic CPUfreq driver for ARM big.LITTLE platforms. + This adds the CPUFreq driver for Airoha EN7581 SoCs. -config ARM_DT_BL_CPUFREQ - tristate "Generic probing via DT for ARM big LITTLE CPUfreq driver" - depends on ARM_BIG_LITTLE_CPUFREQ && OF +config ARM_APPLE_SOC_CPUFREQ + tristate "Apple Silicon SoC CPUFreq support" + depends on ARCH_APPLE || (COMPILE_TEST && 64BIT) + select PM_OPP help - This enables probing via DT for Generic CPUfreq driver for ARM - big.LITTLE platform. This gets frequency tables from DT. + This adds the CPUFreq driver for Apple Silicon machines + (e.g. Apple M1). -config ARM_EXYNOS_CPUFREQ - bool "SAMSUNG EXYNOS SoCs" - depends on ARCH_EXYNOS - select CPU_FREQ_TABLE - default y +config ARM_ARMADA_37XX_CPUFREQ + tristate "Armada 37xx CPUFreq support" + depends on ARCH_MVEBU || COMPILE_TEST + depends on CPUFREQ_DT help - This adds the CPUFreq driver common part for Samsung - EXYNOS SoCs. + This adds the CPUFreq driver support for Marvell Armada 37xx SoCs. + The Armada 37xx PMU supports 4 frequency and VDD levels. + +config ARM_ARMADA_8K_CPUFREQ + tristate "Armada 8K CPUFreq driver" + depends on ARCH_MVEBU || COMPILE_TEST + depends on CPUFREQ_DT + select ARMADA_AP_CPU_CLK if COMMON_CLK + help + This enables the CPUFreq driver support for Marvell + Armada8k SOCs. + Armada8K device has the AP806 which supports scaling + to any full integer divider. If in doubt, say N. -config ARM_EXYNOS4210_CPUFREQ - def_bool CPU_EXYNOS4210 +config ARM_SCPI_CPUFREQ + tristate "SCPI based CPUfreq driver" + depends on ARM_SCPI_PROTOCOL && COMMON_CLK_SCPI help - This adds the CPUFreq driver for Samsung EXYNOS4210 - SoC (S5PV310 or S5PC210). + This adds the CPUfreq driver support for ARM platforms using SCPI + protocol for CPU power management. -config ARM_EXYNOS4X12_CPUFREQ - def_bool (SOC_EXYNOS4212 || SOC_EXYNOS4412) - help - This adds the CPUFreq driver for Samsung EXYNOS4X12 - SoC (EXYNOS4212 or EXYNOS4412). + This driver uses SCPI Message Protocol driver to interact with the + firmware providing the CPU DVFS functionality. -config ARM_EXYNOS5250_CPUFREQ - def_bool SOC_EXYNOS5250 +config ARM_VEXPRESS_SPC_CPUFREQ + tristate "Versatile Express SPC based CPUfreq driver" + depends on ARM_CPU_TOPOLOGY && HAVE_CLK + depends on ARCH_VEXPRESS_SPC || COMPILE_TEST + select PM_OPP help - This adds the CPUFreq driver for Samsung EXYNOS5250 - SoC. + This add the CPUfreq driver support for Versatile Express + big.LITTLE platforms using SPC for power management. -config ARM_EXYNOS5440_CPUFREQ - def_bool SOC_EXYNOS5440 - depends on HAVE_CLK && PM_OPP && OF - select CPU_FREQ_TABLE +config ARM_BRCMSTB_AVS_CPUFREQ + tristate "Broadcom STB AVS CPUfreq driver" + depends on (ARCH_BRCMSTB && !ARM_SCMI_CPUFREQ) || COMPILE_TEST + default y if ARCH_BRCMSTB && !ARM_SCMI_CPUFREQ help - This adds the CPUFreq driver for Samsung EXYNOS5440 - SoC. The nature of exynos5440 clock controller is - different than previous exynos controllers so not using - the common exynos framework. + Some Broadcom STB SoCs use a co-processor running proprietary firmware + ("AVS") to handle voltage and frequency scaling. This driver provides + a standard CPUfreq interface to the firmware. + + Say Y, if you have a Broadcom SoC with AVS support for DFS or DVFS. config ARM_HIGHBANK_CPUFREQ tristate "Calxeda Highbank-based" - depends on ARCH_HIGHBANK - select GENERIC_CPUFREQ_CPU0 - select PM_OPP - select REGULATOR - - default m + depends on ARCH_HIGHBANK || COMPILE_TEST + depends on CPUFREQ_DT && REGULATOR && PL320_MBOX + default m if ARCH_HIGHBANK help This adds the CPUFreq driver for Calxeda Highbank SoC based boards. @@ -70,122 +95,92 @@ config ARM_HIGHBANK_CPUFREQ If in doubt, say N. config ARM_IMX6Q_CPUFREQ - tristate "Freescale i.MX6Q cpufreq support" - depends on SOC_IMX6Q + tristate "Freescale i.MX6 cpufreq support" + depends on ARCH_MXC depends on REGULATOR_ANATOP - select CPU_FREQ_TABLE + depends on NVMEM_IMX_OCOTP || COMPILE_TEST + select PM_OPP help - This adds cpufreq driver support for Freescale i.MX6Q SOC. + This adds cpufreq driver support for Freescale i.MX6 series SoCs. If in doubt, say N. -config ARM_INTEGRATOR - tristate "CPUfreq driver for ARM Integrator CPUs" - depends on ARCH_INTEGRATOR - default y +config ARM_IMX_CPUFREQ_DT + tristate "Freescale i.MX8M cpufreq support" + depends on CPUFREQ_DT + depends on ARCH_MXC || COMPILE_TEST help - This enables the CPUfreq driver for ARM Integrator CPUs. - If in doubt, say Y. + This adds cpufreq driver support for Freescale i.MX7/i.MX8M + series SoCs, based on cpufreq-dt. + + If in doubt, say N. config ARM_KIRKWOOD_CPUFREQ - def_bool ARCH_KIRKWOOD && OF - select CPU_FREQ_TABLE + def_bool MACH_KIRKWOOD help This adds the CPUFreq driver for Marvell Kirkwood SoCs. -config ARM_OMAP2PLUS_CPUFREQ - bool "TI OMAP2+" - depends on ARCH_OMAP2PLUS - default ARCH_OMAP2PLUS - select CPU_FREQ_TABLE - -config ARM_S3C_CPUFREQ - bool - help - Internal configuration node for common cpufreq on Samsung SoC - -config ARM_S3C24XX_CPUFREQ - bool "CPUfreq driver for Samsung S3C24XX series CPUs (EXPERIMENTAL)" - depends on ARCH_S3C24XX - select ARM_S3C_CPUFREQ - help - This enables the CPUfreq driver for the Samsung S3C24XX family - of CPUs. - - For details, take a look at <file:Documentation/cpu-freq>. - - If in doubt, say N. - -config ARM_S3C24XX_CPUFREQ_DEBUG - bool "Debug CPUfreq Samsung driver core" - depends on ARM_S3C24XX_CPUFREQ - help - Enable s3c_freq_dbg for the Samsung S3C CPUfreq core - -config ARM_S3C24XX_CPUFREQ_IODEBUG - bool "Debug CPUfreq Samsung driver IO timing" - depends on ARM_S3C24XX_CPUFREQ - help - Enable s3c_freq_iodbg for the Samsung S3C CPUfreq core - -config ARM_S3C24XX_CPUFREQ_DEBUGFS - bool "Export debugfs for CPUFreq" - depends on ARM_S3C24XX_CPUFREQ && DEBUG_FS +config ARM_MEDIATEK_CPUFREQ + tristate "CPU Frequency scaling support for MediaTek SoCs" + depends on ARCH_MEDIATEK || COMPILE_TEST + depends on REGULATOR + select PM_OPP help - Export status information via debugfs. + This adds the CPUFreq driver support for MediaTek SoCs. -config ARM_S3C2410_CPUFREQ - bool - depends on ARM_S3C24XX_CPUFREQ && CPU_S3C2410 - select S3C2410_CPUFREQ_UTILS +config ARM_MEDIATEK_CPUFREQ_HW + tristate "MediaTek CPUFreq HW driver" + depends on ARCH_MEDIATEK || COMPILE_TEST + default m if ARCH_MEDIATEK help - CPU Frequency scaling support for S3C2410 + Support for the CPUFreq HW driver. + Some MediaTek chipsets have a HW engine to offload the steps + necessary for changing the frequency of the CPUs. Firmware loaded + in this engine exposes a programming interface to the OS. + The driver implements the cpufreq interface for this HW engine. + Say Y if you want to support CPUFreq HW. -config ARM_S3C2412_CPUFREQ - bool - depends on ARM_S3C24XX_CPUFREQ && CPU_S3C2412 - default y - select S3C2412_IOTIMING - help - CPU Frequency scaling support for S3C2412 and S3C2413 SoC CPUs. +config ARM_OMAP2PLUS_CPUFREQ + bool "TI OMAP2+" + depends on ARCH_OMAP2PLUS || COMPILE_TEST + default ARCH_OMAP2PLUS -config ARM_S3C2416_CPUFREQ - bool "S3C2416 CPU Frequency scaling support" - depends on CPU_S3C2416 - select CPU_FREQ_TABLE +config ARM_QCOM_CPUFREQ_NVMEM + tristate "Qualcomm nvmem based CPUFreq" + depends on ARCH_QCOM || COMPILE_TEST + depends on NVMEM_QCOM_QFPROM + depends on QCOM_SMEM + select PM_OPP help - This adds the CPUFreq driver for the Samsung S3C2416 and - S3C2450 SoC. The S3C2416 supports changing the rate of the - armdiv clock source and also entering a so called dynamic - voltage scaling mode in which it is possible to reduce the - core voltage of the cpu. + This adds the CPUFreq driver for Qualcomm Kryo SoC based boards. If in doubt, say N. -config ARM_S3C2416_CPUFREQ_VCORESCALE - bool "Allow voltage scaling for S3C2416 arm core" - depends on ARM_S3C2416_CPUFREQ && REGULATOR +config ARM_QCOM_CPUFREQ_HW + tristate "QCOM CPUFreq HW driver" + depends on ARCH_QCOM || COMPILE_TEST + depends on COMMON_CLK help - Enable CPU voltage scaling when entering the dvs mode. - It uses information gathered through existing hardware and - tests but not documented in any datasheet. + Support for the CPUFreq HW driver. + Some QCOM chipsets have a HW engine to offload the steps + necessary for changing the frequency of the CPUs. Firmware loaded + in this engine exposes a programming interface to the OS. + The driver implements the cpufreq interface for this HW engine. + Say Y if you want to support CPUFreq HW. + +config ARM_RASPBERRYPI_CPUFREQ + tristate "Raspberry Pi cpufreq support" + depends on CLK_RASPBERRYPI || COMPILE_TEST + help + This adds the CPUFreq driver for Raspberry Pi If in doubt, say N. -config ARM_S3C2440_CPUFREQ - bool "S3C2440/S3C2442 CPU Frequency scaling support" - depends on ARM_S3C24XX_CPUFREQ && (CPU_S3C2440 || CPU_S3C2442) - select S3C2410_CPUFREQ_UTILS - default y - help - CPU Frequency scaling support for S3C2440 and S3C2442 SoC CPUs. - config ARM_S3C64XX_CPUFREQ bool "Samsung S3C64XX" - depends on CPU_S3C6410 - select CPU_FREQ_TABLE - default y + depends on CPU_S3C6410 || COMPILE_TEST + default CPU_S3C6410 help This adds the CPUFreq driver for Samsung S3C6410 SoC. @@ -193,33 +188,93 @@ config ARM_S3C64XX_CPUFREQ config ARM_S5PV210_CPUFREQ bool "Samsung S5PV210 and S5PC110" - depends on CPU_S5PV210 - select CPU_FREQ_TABLE - default y + depends on CPU_S5PV210 || COMPILE_TEST + default CPU_S5PV210 help This adds the CPUFreq driver for Samsung S5PV210 and S5PC110 SoCs. If in doubt, say N. -config ARM_SA1100_CPUFREQ - bool - config ARM_SA1110_CPUFREQ bool +config ARM_SCMI_CPUFREQ + tristate "SCMI based CPUfreq driver" + depends on ARM_SCMI_PROTOCOL || COMPILE_TEST + select PM_OPP + help + This adds the CPUfreq driver support for ARM platforms using SCMI + protocol for CPU power management. + + This driver uses SCMI Message Protocol driver to interact with the + firmware providing the CPU DVFS functionality. + config ARM_SPEAR_CPUFREQ bool "SPEAr CPUFreq support" - depends on PLAT_SPEAR - select CPU_FREQ_TABLE - default y + depends on PLAT_SPEAR || COMPILE_TEST + default PLAT_SPEAR help This adds the CPUFreq driver support for SPEAr SOCs. -config ARM_TEGRA_CPUFREQ - bool "TEGRA CPUFreq support" - depends on ARCH_TEGRA - select CPU_FREQ_TABLE - default y +config ARM_STI_CPUFREQ + tristate "STi CPUFreq support" + depends on CPUFREQ_DT + depends on SOC_STIH407 || COMPILE_TEST + help + This driver uses the generic OPP framework to match the running + platform with a predefined set of suitable values. If not provided + we will fall-back so safe-values contained in Device Tree. Enable + this config option if you wish to add CPUFreq support for STi based + SoCs. + +config ARM_TEGRA20_CPUFREQ + tristate "Tegra20/30 CPUFreq support" + depends on ARCH_TEGRA || COMPILE_TEST + depends on CPUFREQ_DT + default ARCH_TEGRA + help + This adds the CPUFreq driver support for Tegra20/30 SOCs. + +config ARM_TEGRA124_CPUFREQ + tristate "Tegra124 CPUFreq support" + depends on ARCH_TEGRA || COMPILE_TEST + depends on CPUFREQ_DT + default ARCH_TEGRA + help + This adds the CPUFreq driver support for Tegra124 SOCs. + +config ARM_TEGRA186_CPUFREQ + tristate "Tegra186 CPUFreq support" + depends on ARCH_TEGRA || COMPILE_TEST + depends on TEGRA_BPMP help - This adds the CPUFreq driver support for TEGRA SOCs. + This adds the CPUFreq driver support for Tegra186 SOCs. + +config ARM_TEGRA194_CPUFREQ + tristate "Tegra194 CPUFreq support" + depends on ARCH_TEGRA_194_SOC || ARCH_TEGRA_234_SOC || (64BIT && COMPILE_TEST) + depends on TEGRA_BPMP + default ARCH_TEGRA_194_SOC || ARCH_TEGRA_234_SOC + help + This adds CPU frequency driver support for Tegra194 SOCs. + +config ARM_TI_CPUFREQ + bool "Texas Instruments CPUFreq support" + depends on ARCH_OMAP2PLUS || ARCH_K3 || COMPILE_TEST + default ARCH_OMAP2PLUS || ARCH_K3 + help + This driver enables valid OPPs on the running platform based on + values contained within the SoC in use. Enable this in order to + use the cpufreq-dt driver on all Texas Instruments platforms that + provide dt based operating-points-v2 tables with opp-supported-hw + data provided. Required for cpufreq support on AM335x, AM437x, + DRA7x, and AM57x platforms. + +config ARM_PXA2xx_CPUFREQ + tristate "Intel PXA2xx CPUfreq driver" + depends on PXA27x || PXA25x || COMPILE_TEST + help + This add the CPUFreq driver support for Intel PXA2xx SOCs. + + If in doubt, say N. diff --git a/drivers/cpufreq/Kconfig.powerpc b/drivers/cpufreq/Kconfig.powerpc index 25ca9db62e09..551e65d35a1d 100644 --- a/drivers/cpufreq/Kconfig.powerpc +++ b/drivers/cpufreq/Kconfig.powerpc @@ -1,44 +1,7 @@ -config CPU_FREQ_CBE - tristate "CBE frequency scaling" - depends on CBE_RAS && PPC_CELL - select CPU_FREQ_TABLE - default m - help - This adds the cpufreq driver for Cell BE processors. - For details, take a look at <file:Documentation/cpu-freq/>. - If you don't have such processor, say N - -config CPU_FREQ_CBE_PMI - bool "CBE frequency scaling using PMI interface" - depends on CPU_FREQ_CBE - default n - help - Select this, if you want to use the PMI interface to switch - frequencies. Using PMI, the processor will not only be able to run at - lower speed, but also at lower core voltage. - -config CPU_FREQ_MAPLE - bool "Support for Maple 970FX Evaluation Board" - depends on PPC_MAPLE - select CPU_FREQ_TABLE - help - This adds support for frequency switching on Maple 970FX - Evaluation Board and compatible boards (IBM JS2x blades). - -config PPC_CORENET_CPUFREQ - tristate "CPU frequency scaling driver for Freescale E500MC SoCs" - depends on PPC_E500MC && OF && COMMON_CLK - select CPU_FREQ_TABLE - select CLK_PPC_CORENET - help - This adds the CPUFreq driver support for Freescale e500mc, - e5500 and e6500 series SoCs which are capable of changing - the CPU's frequency dynamically. - +# SPDX-License-Identifier: GPL-2.0-only config CPU_FREQ_PMAC bool "Support for Apple PowerBooks" depends on ADB_PMU && PPC32 - select CPU_FREQ_TABLE help This adds support for frequency switching on Apple PowerBooks, this currently includes some models of iBook & Titanium @@ -47,7 +10,6 @@ config CPU_FREQ_PMAC config CPU_FREQ_PMAC64 bool "Support for some Apple G5s" depends on PPC_PMAC && PPC64 - select CPU_FREQ_TABLE help This adds support for frequency switching on Apple iMac G5, and some of the more recent desktop G5 machines as well. @@ -55,8 +17,15 @@ config CPU_FREQ_PMAC64 config PPC_PASEMI_CPUFREQ bool "Support for PA Semi PWRficient" depends on PPC_PASEMI - select CPU_FREQ_TABLE default y help This adds the support for frequency switching on PA Semi PWRficient processors. + +config POWERNV_CPUFREQ + tristate "CPU frequency scaling for IBM POWERNV platform" + depends on PPC_POWERNV + default y + help + This adds support for CPU frequency switching on IBM POWERNV + platform diff --git a/drivers/cpufreq/Kconfig.x86 b/drivers/cpufreq/Kconfig.x86 index e2b6eabef221..2c5c228408bf 100644 --- a/drivers/cpufreq/Kconfig.x86 +++ b/drivers/cpufreq/Kconfig.x86 @@ -1,17 +1,22 @@ +# SPDX-License-Identifier: GPL-2.0-only # # x86 CPU Frequency scaling drivers # config X86_INTEL_PSTATE - bool "Intel P state control" - depends on X86 - help - This driver provides a P state for Intel core processors. + bool "Intel P state control" + depends on X86 + select ACPI_PROCESSOR if ACPI + select ACPI_CPPC_LIB if X86_64 && ACPI && SCHED_MC_PRIO + select CPU_FREQ_GOV_PERFORMANCE + select CPU_FREQ_GOV_SCHEDUTIL if SMP + help + This driver provides a P state for Intel core processors. The driver implements an internal governor and will become - the scaling driver and governor for Sandy bridge processors. + the scaling driver and governor for Sandy bridge processors. - When this driver is enabled it will become the perferred - scaling driver for Sandy bridge processors. + When this driver is enabled it will become the preferred + scaling driver for Sandy bridge processors. If in doubt, say N. @@ -22,16 +27,65 @@ config X86_PCC_CPUFREQ This driver adds support for the PCC interface. For details, take a look at: - <file:Documentation/cpu-freq/pcc-cpufreq.txt>. + <file:Documentation/admin-guide/pm/cpufreq_drivers.rst>. To compile this driver as a module, choose M here: the module will be called pcc-cpufreq. If in doubt, say N. +config X86_AMD_PSTATE + bool "AMD Processor P-State driver" + depends on X86 && ACPI + select ACPI_PROCESSOR + select ACPI_CPPC_LIB if X86_64 + select CPU_FREQ_GOV_SCHEDUTIL if SMP + help + This driver adds a CPUFreq driver which utilizes a fine grain + processor performance frequency control range instead of legacy + performance levels. _CPC needs to be present in the ACPI tables + of the system. + + For details, take a look at: + <file:Documentation/admin-guide/pm/amd-pstate.rst>. + + If in doubt, say N. + +config X86_AMD_PSTATE_DEFAULT_MODE + int "AMD Processor P-State default mode" + depends on X86_AMD_PSTATE + default 3 if X86_AMD_PSTATE + range 1 4 + help + Select the default mode the amd-pstate driver will use on + supported hardware. + The value set has the following meanings: + 1 -> Disabled + 2 -> Passive + 3 -> Active (EPP) + 4 -> Guided + + For details, take a look at: + <file:Documentation/admin-guide/pm/amd-pstate.rst>. + +config X86_AMD_PSTATE_UT + tristate "selftest for AMD Processor P-State driver" + depends on X86 && ACPI_PROCESSOR + depends on X86_AMD_PSTATE + default n + help + This kernel module is used for testing. It's safe to say M here. + + It can also be built-in without X86_AMD_PSTATE enabled. + Currently, only tests for amd-pstate are supported. If X86_AMD_PSTATE + is set disabled, it can tell the users test can only run on amd-pstate + driver, please set X86_AMD_PSTATE enabled. + In the future, comparison tests will be added. It can set amd-pstate + disabled and set acpi-cpufreq enabled to run test cases, then compare + the test results. + config X86_ACPI_CPUFREQ tristate "ACPI Processor P-States driver" - select CPU_FREQ_TABLE depends on ACPI_PROCESSOR help This driver adds a CPUFreq driver which utilizes the ACPI @@ -53,16 +107,15 @@ config X86_ACPI_CPUFREQ_CPB help The powernow-k8 driver used to provide a sysfs knob called "cpb" to disable the Core Performance Boosting feature of AMD CPUs. This - file has now been superseeded by the more generic "boost" entry. + file has now been superseded by the more generic "boost" entry. By enabling this option the acpi_cpufreq driver provides the old entry in addition to the new boost ones, for compatibility reasons. config ELAN_CPUFREQ tristate "AMD Elan SC400 and SC410" - select CPU_FREQ_TABLE depends on MELAN - ---help--- + help This adds the CPUFreq driver for AMD Elan SC400 and SC410 processors. @@ -76,9 +129,8 @@ config ELAN_CPUFREQ config SC520_CPUFREQ tristate "AMD Elan SC520" - select CPU_FREQ_TABLE depends on MELAN - ---help--- + help This adds the CPUFreq driver for AMD Elan SC520 processor. For details, take a look at <file:Documentation/cpu-freq/>. @@ -88,7 +140,6 @@ config SC520_CPUFREQ config X86_POWERNOW_K6 tristate "AMD Mobile K6-2/K6-3 PowerNow!" - select CPU_FREQ_TABLE depends on X86_32 help This adds the CPUFreq driver for mobile AMD K6-2+ and mobile @@ -100,7 +151,6 @@ config X86_POWERNOW_K6 config X86_POWERNOW_K7 tristate "AMD Mobile Athlon/Duron PowerNow!" - select CPU_FREQ_TABLE depends on X86_32 help This adds the CPUFreq driver for mobile AMD K7 mobile processors. @@ -118,7 +168,6 @@ config X86_POWERNOW_K7_ACPI config X86_POWERNOW_K8 tristate "AMD Opteron/Athlon64 PowerNow!" - select CPU_FREQ_TABLE depends on ACPI && ACPI_PROCESSOR && X86_ACPI_CPUFREQ help This adds the CPUFreq driver for K8/early Opteron/Athlon64 processors. @@ -132,11 +181,10 @@ config X86_POWERNOW_K8 config X86_AMD_FREQ_SENSITIVITY tristate "AMD frequency sensitivity feedback powersave bias" depends on CPU_FREQ_GOV_ONDEMAND && X86_ACPI_CPUFREQ && CPU_SUP_AMD - select CPU_FREQ_TABLE help This adds AMD-specific powersave bias function to the ondemand governor, which allows it to make more power-conscious frequency - change decisions based on feedback from hardware (availble on AMD + change decisions based on feedback from hardware (available on AMD Family 16h and above). Hardware feedback tells software how "sensitive" to frequency changes @@ -160,7 +208,6 @@ config X86_GX_SUSPMOD config X86_SPEEDSTEP_CENTRINO tristate "Intel Enhanced SpeedStep (deprecated)" - select CPU_FREQ_TABLE select X86_SPEEDSTEP_CENTRINO_TABLE if X86_32 depends on X86_32 || (X86_64 && ACPI_PROCESSOR) help @@ -190,7 +237,6 @@ config X86_SPEEDSTEP_CENTRINO_TABLE config X86_SPEEDSTEP_ICH tristate "Intel Speedstep on ICH-M chipsets (ioport interface)" - select CPU_FREQ_TABLE depends on X86_32 help This adds the CPUFreq driver for certain mobile Intel Pentium III @@ -204,7 +250,6 @@ config X86_SPEEDSTEP_ICH config X86_SPEEDSTEP_SMI tristate "Intel SpeedStep on 440BX/ZX/MX chipsets (SMI interface)" - select CPU_FREQ_TABLE depends on X86_32 help This adds the CPUFreq driver for certain mobile Intel Pentium III @@ -217,7 +262,6 @@ config X86_SPEEDSTEP_SMI config X86_P4_CLOCKMOD tristate "Intel Pentium 4 clock modulation" - select CPU_FREQ_TABLE help This adds the CPUFreq driver for Intel Pentium 4 / XEON processors. When enabled it will lower CPU temperature by skipping @@ -259,7 +303,6 @@ config X86_LONGRUN config X86_LONGHAUL tristate "VIA Cyrix III Longhaul" - select CPU_FREQ_TABLE depends on X86_32 && ACPI_PROCESSOR help This adds the CPUFreq driver for VIA Samuel/CyrixIII, @@ -272,7 +315,6 @@ config X86_LONGHAUL config X86_E_POWERSAVER tristate "VIA C7 Enhanced PowerSaver (DANGEROUS)" - select CPU_FREQ_TABLE depends on X86_32 && ACPI_PROCESSOR help This adds the CPUFreq driver for VIA C7 processors. However, this driver @@ -298,3 +340,15 @@ config X86_SPEEDSTEP_RELAXED_CAP_CHECK option lets the probing code bypass some of those checks if the parameter "relaxed_check=1" is passed to the module. +config CPUFREQ_ARCH_CUR_FREQ + default y + bool "Current frequency derived from HW provided feedback" + help + This determines whether the scaling_cur_freq sysfs attribute returns + the last requested frequency or a more precise value based on hardware + provided feedback (as architected counters). + Given that a more precise frequency can now be provided via the + cpuinfo_avg_freq attribute, by enabling this option, + scaling_cur_freq maintains the provision of a counter based frequency, + for compatibility reasons. + diff --git a/drivers/cpufreq/Makefile b/drivers/cpufreq/Makefile index d345b5a7aa71..681d687b5a18 100644 --- a/drivers/cpufreq/Makefile +++ b/drivers/cpufreq/Makefile @@ -1,5 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0 # CPUfreq core -obj-$(CONFIG_CPU_FREQ) += cpufreq.o +obj-$(CONFIG_CPU_FREQ) += cpufreq.o freq_table.o + # CPUfreq stats obj-$(CONFIG_CPU_FREQ_STAT) += cpufreq_stats.o @@ -10,11 +12,17 @@ obj-$(CONFIG_CPU_FREQ_GOV_USERSPACE) += cpufreq_userspace.o obj-$(CONFIG_CPU_FREQ_GOV_ONDEMAND) += cpufreq_ondemand.o obj-$(CONFIG_CPU_FREQ_GOV_CONSERVATIVE) += cpufreq_conservative.o obj-$(CONFIG_CPU_FREQ_GOV_COMMON) += cpufreq_governor.o +obj-$(CONFIG_CPU_FREQ_GOV_ATTR_SET) += cpufreq_governor_attr_set.o -# CPUfreq cross-arch helpers -obj-$(CONFIG_CPU_FREQ_TABLE) += freq_table.o +obj-$(CONFIG_CPUFREQ_DT) += cpufreq-dt.o +obj-$(CONFIG_CPUFREQ_DT_RUST) += rcpufreq_dt.o +obj-$(CONFIG_CPUFREQ_DT_PLATDEV) += cpufreq-dt-platdev.o +obj-$(CONFIG_CPUFREQ_VIRT) += virtual-cpufreq.o -obj-$(CONFIG_GENERIC_CPUFREQ_CPU0) += cpufreq-cpu0.o +# Traces +CFLAGS_amd-pstate-trace.o := -I$(src) +CFLAGS_powernv-cpufreq.o := -I$(src) +amd_pstate-y := amd-pstate.o amd-pstate-trace.o ################################################################################## # x86 drivers. @@ -23,7 +31,9 @@ obj-$(CONFIG_GENERIC_CPUFREQ_CPU0) += cpufreq-cpu0.o # powernow-k8 can load then. ACPI is preferred to all other hardware-specific drivers. # speedstep-* is preferred over p4-clockmod. -obj-$(CONFIG_X86_ACPI_CPUFREQ) += acpi-cpufreq.o mperf.o +obj-$(CONFIG_X86_ACPI_CPUFREQ) += acpi-cpufreq.o +obj-$(CONFIG_X86_AMD_PSTATE) += amd_pstate.o +obj-$(CONFIG_X86_AMD_PSTATE_UT) += amd-pstate-ut.o obj-$(CONFIG_X86_POWERNOW_K8) += powernow-k8.o obj-$(CONFIG_X86_PCC_CPUFREQ) += pcc-cpufreq.o obj-$(CONFIG_X86_POWERNOW_K6) += powernow-k6.o @@ -45,59 +55,55 @@ obj-$(CONFIG_X86_AMD_FREQ_SENSITIVITY) += amd_freq_sensitivity.o ################################################################################## # ARM SoC drivers -obj-$(CONFIG_ARM_BIG_LITTLE_CPUFREQ) += arm_big_little.o -# big LITTLE per platform glues. Keep DT_BL_CPUFREQ as the last entry in all big -# LITTLE drivers, so that it is probed last. -obj-$(CONFIG_ARM_DT_BL_CPUFREQ) += arm_big_little_dt.o - -obj-$(CONFIG_ARCH_DAVINCI_DA850) += davinci-cpufreq.o -obj-$(CONFIG_UX500_SOC_DB8500) += dbx500-cpufreq.o -obj-$(CONFIG_ARM_EXYNOS_CPUFREQ) += exynos-cpufreq.o -obj-$(CONFIG_ARM_EXYNOS4210_CPUFREQ) += exynos4210-cpufreq.o -obj-$(CONFIG_ARM_EXYNOS4X12_CPUFREQ) += exynos4x12-cpufreq.o -obj-$(CONFIG_ARM_EXYNOS5250_CPUFREQ) += exynos5250-cpufreq.o -obj-$(CONFIG_ARM_EXYNOS5440_CPUFREQ) += exynos5440-cpufreq.o +obj-$(CONFIG_ARM_AIROHA_SOC_CPUFREQ) += airoha-cpufreq.o +obj-$(CONFIG_ARM_APPLE_SOC_CPUFREQ) += apple-soc-cpufreq.o +obj-$(CONFIG_ARM_ARMADA_37XX_CPUFREQ) += armada-37xx-cpufreq.o +obj-$(CONFIG_ARM_ARMADA_8K_CPUFREQ) += armada-8k-cpufreq.o +obj-$(CONFIG_ARM_BRCMSTB_AVS_CPUFREQ) += brcmstb-avs-cpufreq.o +obj-$(CONFIG_ACPI_CPPC_CPUFREQ) += cppc_cpufreq.o +obj-$(CONFIG_ARCH_DAVINCI) += davinci-cpufreq.o obj-$(CONFIG_ARM_HIGHBANK_CPUFREQ) += highbank-cpufreq.o obj-$(CONFIG_ARM_IMX6Q_CPUFREQ) += imx6q-cpufreq.o -obj-$(CONFIG_ARM_INTEGRATOR) += integrator-cpufreq.o +obj-$(CONFIG_ARM_IMX_CPUFREQ_DT) += imx-cpufreq-dt.o obj-$(CONFIG_ARM_KIRKWOOD_CPUFREQ) += kirkwood-cpufreq.o +obj-$(CONFIG_ARM_MEDIATEK_CPUFREQ) += mediatek-cpufreq.o +obj-$(CONFIG_ARM_MEDIATEK_CPUFREQ_HW) += mediatek-cpufreq-hw.o +obj-$(CONFIG_MACH_MVEBU_V7) += mvebu-cpufreq.o obj-$(CONFIG_ARM_OMAP2PLUS_CPUFREQ) += omap-cpufreq.o -obj-$(CONFIG_PXA25x) += pxa2xx-cpufreq.o -obj-$(CONFIG_PXA27x) += pxa2xx-cpufreq.o +obj-$(CONFIG_ARM_PXA2xx_CPUFREQ) += pxa2xx-cpufreq.o obj-$(CONFIG_PXA3xx) += pxa3xx-cpufreq.o -obj-$(CONFIG_ARM_S3C24XX_CPUFREQ) += s3c24xx-cpufreq.o -obj-$(CONFIG_ARM_S3C24XX_CPUFREQ_DEBUGFS) += s3c24xx-cpufreq-debugfs.o -obj-$(CONFIG_ARM_S3C2410_CPUFREQ) += s3c2410-cpufreq.o -obj-$(CONFIG_ARM_S3C2412_CPUFREQ) += s3c2412-cpufreq.o -obj-$(CONFIG_ARM_S3C2416_CPUFREQ) += s3c2416-cpufreq.o -obj-$(CONFIG_ARM_S3C2440_CPUFREQ) += s3c2440-cpufreq.o +obj-$(CONFIG_ARM_QCOM_CPUFREQ_HW) += qcom-cpufreq-hw.o +obj-$(CONFIG_ARM_QCOM_CPUFREQ_NVMEM) += qcom-cpufreq-nvmem.o +obj-$(CONFIG_ARM_RASPBERRYPI_CPUFREQ) += raspberrypi-cpufreq.o obj-$(CONFIG_ARM_S3C64XX_CPUFREQ) += s3c64xx-cpufreq.o obj-$(CONFIG_ARM_S5PV210_CPUFREQ) += s5pv210-cpufreq.o -obj-$(CONFIG_ARM_SA1100_CPUFREQ) += sa1100-cpufreq.o obj-$(CONFIG_ARM_SA1110_CPUFREQ) += sa1110-cpufreq.o +obj-$(CONFIG_ARM_SCMI_CPUFREQ) += scmi-cpufreq.o +obj-$(CONFIG_ARM_SCPI_CPUFREQ) += scpi-cpufreq.o obj-$(CONFIG_ARM_SPEAR_CPUFREQ) += spear-cpufreq.o -obj-$(CONFIG_ARM_TEGRA_CPUFREQ) += tegra-cpufreq.o +obj-$(CONFIG_ARM_STI_CPUFREQ) += sti-cpufreq.o +obj-$(CONFIG_ARM_ALLWINNER_SUN50I_CPUFREQ_NVMEM) += sun50i-cpufreq-nvmem.o +obj-$(CONFIG_ARM_TEGRA20_CPUFREQ) += tegra20-cpufreq.o +obj-$(CONFIG_ARM_TEGRA124_CPUFREQ) += tegra124-cpufreq.o +obj-$(CONFIG_ARM_TEGRA186_CPUFREQ) += tegra186-cpufreq.o +obj-$(CONFIG_ARM_TEGRA194_CPUFREQ) += tegra194-cpufreq.o +obj-$(CONFIG_ARM_TI_CPUFREQ) += ti-cpufreq.o +obj-$(CONFIG_ARM_VEXPRESS_SPC_CPUFREQ) += vexpress-spc-cpufreq.o + ################################################################################## # PowerPC platform drivers -obj-$(CONFIG_CPU_FREQ_CBE) += ppc-cbe-cpufreq.o -ppc-cbe-cpufreq-y += ppc_cbe_cpufreq_pervasive.o ppc_cbe_cpufreq.o -obj-$(CONFIG_CPU_FREQ_CBE_PMI) += ppc_cbe_cpufreq_pmi.o -obj-$(CONFIG_CPU_FREQ_MAPLE) += maple-cpufreq.o -obj-$(CONFIG_PPC_CORENET_CPUFREQ) += ppc-corenet-cpufreq.o +obj-$(CONFIG_QORIQ_CPUFREQ) += qoriq-cpufreq.o obj-$(CONFIG_CPU_FREQ_PMAC) += pmac32-cpufreq.o obj-$(CONFIG_CPU_FREQ_PMAC64) += pmac64-cpufreq.o obj-$(CONFIG_PPC_PASEMI_CPUFREQ) += pasemi-cpufreq.o +obj-$(CONFIG_POWERNV_CPUFREQ) += powernv-cpufreq.o ################################################################################## # Other platform drivers -obj-$(CONFIG_AVR32_AT32AP_CPUFREQ) += at32ap-cpufreq.o -obj-$(CONFIG_BFIN_CPU_FREQ) += blackfin-cpufreq.o -obj-$(CONFIG_CRIS_MACH_ARTPEC3) += cris-artpec3-cpufreq.o -obj-$(CONFIG_ETRAXFS) += cris-etraxfs-cpufreq.o -obj-$(CONFIG_IA64_ACPI_CPUFREQ) += ia64-acpi-cpufreq.o +obj-$(CONFIG_BMIPS_CPUFREQ) += bmips-cpufreq.o obj-$(CONFIG_LOONGSON2_CPUFREQ) += loongson2_cpufreq.o +obj-$(CONFIG_LOONGSON3_CPUFREQ) += loongson3_cpufreq.o obj-$(CONFIG_SH_CPU_FREQ) += sh-cpufreq.o obj-$(CONFIG_SPARC_US2E_CPUFREQ) += sparc-us2e-cpufreq.o obj-$(CONFIG_SPARC_US3_CPUFREQ) += sparc-us3-cpufreq.o -obj-$(CONFIG_UNICORE32) += unicore2-cpufreq.o diff --git a/drivers/cpufreq/acpi-cpufreq.c b/drivers/cpufreq/acpi-cpufreq.c index 39264020b88a..e73a66785d69 100644 --- a/drivers/cpufreq/acpi-cpufreq.c +++ b/drivers/cpufreq/acpi-cpufreq.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-or-later /* * acpi-cpufreq.c - ACPI Processor P-States Driver * @@ -5,26 +6,10 @@ * Copyright (C) 2001, 2002 Paul Diefenbaugh <paul.s.diefenbaugh@intel.com> * Copyright (C) 2002 - 2004 Dominik Brodowski <linux@brodo.de> * Copyright (C) 2006 Denis Sadykov <denis.m.sadykov@intel.com> - * - * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or (at - * your option) any later version. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - * - * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + #include <linux/kernel.h> #include <linux/module.h> #include <linux/init.h> @@ -34,6 +19,8 @@ #include <linux/compiler.h> #include <linux/dmi.h> #include <linux/slab.h> +#include <linux/string_helpers.h> +#include <linux/platform_device.h> #include <linux/acpi.h> #include <linux/io.h> @@ -41,18 +28,17 @@ #include <linux/uaccess.h> #include <acpi/processor.h> +#include <acpi/cppc_acpi.h> #include <asm/msr.h> #include <asm/processor.h> #include <asm/cpufeature.h> -#include "mperf.h" +#include <asm/cpu_device_id.h> MODULE_AUTHOR("Paul Diefenbaugh, Dominik Brodowski"); MODULE_DESCRIPTION("ACPI Processor P-States Driver"); MODULE_LICENSE("GPL"); -#define PFX "acpi-cpufreq: " - enum { UNDEFINED_CAPABLE = 0, SYSTEM_INTEL_MSR_CAPABLE, @@ -62,124 +48,102 @@ enum { #define INTEL_MSR_RANGE (0xffff) #define AMD_MSR_RANGE (0x7) - -#define MSR_K7_HWCR_CPB_DIS (1ULL << 25) +#define HYGON_MSR_RANGE (0x7) struct acpi_cpufreq_data { - struct acpi_processor_performance *acpi_data; - struct cpufreq_frequency_table *freq_table; unsigned int resume; unsigned int cpu_feature; + unsigned int acpi_perf_cpu; cpumask_var_t freqdomain_cpus; + void (*cpu_freq_write)(struct acpi_pct_register *reg, u32 val); + u32 (*cpu_freq_read)(struct acpi_pct_register *reg); }; -static DEFINE_PER_CPU(struct acpi_cpufreq_data *, acfreq_data); - /* acpi_perf_data is a pointer to percpu data. */ static struct acpi_processor_performance __percpu *acpi_perf_data; +static inline struct acpi_processor_performance *to_perf_data(struct acpi_cpufreq_data *data) +{ + return per_cpu_ptr(acpi_perf_data, data->acpi_perf_cpu); +} + static struct cpufreq_driver acpi_cpufreq_driver; static unsigned int acpi_pstate_strict; -static bool boost_enabled, boost_supported; -static struct msr __percpu *msrs; static bool boost_state(unsigned int cpu) { - u32 lo, hi; u64 msr; switch (boot_cpu_data.x86_vendor) { case X86_VENDOR_INTEL: - rdmsr_on_cpu(cpu, MSR_IA32_MISC_ENABLE, &lo, &hi); - msr = lo | ((u64)hi << 32); + case X86_VENDOR_CENTAUR: + case X86_VENDOR_ZHAOXIN: + rdmsrq_on_cpu(cpu, MSR_IA32_MISC_ENABLE, &msr); return !(msr & MSR_IA32_MISC_ENABLE_TURBO_DISABLE); + case X86_VENDOR_HYGON: case X86_VENDOR_AMD: - rdmsr_on_cpu(cpu, MSR_K7_HWCR, &lo, &hi); - msr = lo | ((u64)hi << 32); + rdmsrq_on_cpu(cpu, MSR_K7_HWCR, &msr); return !(msr & MSR_K7_HWCR_CPB_DIS); } return false; } -static void boost_set_msrs(bool enable, const struct cpumask *cpumask) +static int boost_set_msr(bool enable) { - u32 cpu; u32 msr_addr; - u64 msr_mask; + u64 msr_mask, val; switch (boot_cpu_data.x86_vendor) { case X86_VENDOR_INTEL: + case X86_VENDOR_CENTAUR: + case X86_VENDOR_ZHAOXIN: msr_addr = MSR_IA32_MISC_ENABLE; msr_mask = MSR_IA32_MISC_ENABLE_TURBO_DISABLE; break; + case X86_VENDOR_HYGON: case X86_VENDOR_AMD: msr_addr = MSR_K7_HWCR; msr_mask = MSR_K7_HWCR_CPB_DIS; break; default: - return; + return -EINVAL; } - rdmsr_on_cpus(cpumask, msr_addr, msrs); + rdmsrq(msr_addr, val); - for_each_cpu(cpu, cpumask) { - struct msr *reg = per_cpu_ptr(msrs, cpu); - if (enable) - reg->q &= ~msr_mask; - else - reg->q |= msr_mask; - } + if (enable) + val &= ~msr_mask; + else + val |= msr_mask; - wrmsr_on_cpus(cpumask, msr_addr, msrs); + wrmsrq(msr_addr, val); + return 0; } -static ssize_t _store_boost(const char *buf, size_t count) +static void boost_set_msr_each(void *p_en) { - int ret; - unsigned long val = 0; - - if (!boost_supported) - return -EINVAL; - - ret = kstrtoul(buf, 10, &val); - if (ret || (val > 1)) - return -EINVAL; - - if ((val && boost_enabled) || (!val && !boost_enabled)) - return count; - - get_online_cpus(); + bool enable = (bool) p_en; - boost_set_msrs(val, cpu_online_mask); - - put_online_cpus(); - - boost_enabled = val; - pr_debug("Core Boosting %sabled.\n", val ? "en" : "dis"); - - return count; + boost_set_msr(enable); } -static ssize_t store_global_boost(struct kobject *kobj, struct attribute *attr, - const char *buf, size_t count) +static int set_boost(struct cpufreq_policy *policy, int val) { - return _store_boost(buf, count); -} + on_each_cpu_mask(policy->cpus, boost_set_msr_each, + (void *)(long)val, 1); + pr_debug("CPU %*pbl: Core Boosting %s.\n", + cpumask_pr_args(policy->cpus), str_enabled_disabled(val)); -static ssize_t show_global_boost(struct kobject *kobj, - struct attribute *attr, char *buf) -{ - return sprintf(buf, "%u\n", boost_enabled); + return 0; } -static struct global_attr global_boost = __ATTR(boost, 0644, - show_global_boost, - store_global_boost); - static ssize_t show_freqdomain_cpus(struct cpufreq_policy *policy, char *buf) { - struct acpi_cpufreq_data *data = per_cpu(acfreq_data, policy->cpu); + struct acpi_cpufreq_data *data = policy->driver_data; + + if (unlikely(!data)) + return -ENODEV; return cpufreq_show_cpus(data->freqdomain_cpus, buf); } @@ -190,15 +154,29 @@ cpufreq_freq_attr_ro(freqdomain_cpus); static ssize_t store_cpb(struct cpufreq_policy *policy, const char *buf, size_t count) { - return _store_boost(buf, count); + int ret; + unsigned int val = 0; + + if (!acpi_cpufreq_driver.set_boost) + return -EINVAL; + + ret = kstrtouint(buf, 10, &val); + if (ret || val > 1) + return -EINVAL; + + cpus_read_lock(); + set_boost(policy, val); + cpus_read_unlock(); + + return count; } static ssize_t show_cpb(struct cpufreq_policy *policy, char *buf) { - return sprintf(buf, "%u\n", boost_enabled); + return sprintf(buf, "%u\n", acpi_cpufreq_driver.boost_enabled); } -static struct freq_attr cpb = __ATTR(cpb, 0644, show_cpb, store_cpb); +cpufreq_freq_attr_rw(cpb); #endif static int check_est_cpu(unsigned int cpuid) @@ -215,187 +193,184 @@ static int check_amd_hwpstate_cpu(unsigned int cpuid) return cpu_has(cpu, X86_FEATURE_HW_PSTATE); } -static unsigned extract_io(u32 value, struct acpi_cpufreq_data *data) +static unsigned extract_io(struct cpufreq_policy *policy, u32 value) { + struct acpi_cpufreq_data *data = policy->driver_data; struct acpi_processor_performance *perf; int i; - perf = data->acpi_data; + perf = to_perf_data(data); for (i = 0; i < perf->state_count; i++) { if (value == perf->states[i].status) - return data->freq_table[i].frequency; + return policy->freq_table[i].frequency; } return 0; } -static unsigned extract_msr(u32 msr, struct acpi_cpufreq_data *data) +static unsigned extract_msr(struct cpufreq_policy *policy, u32 msr) { - int i; + struct acpi_cpufreq_data *data = policy->driver_data; + struct cpufreq_frequency_table *pos; struct acpi_processor_performance *perf; if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) msr &= AMD_MSR_RANGE; + else if (boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) + msr &= HYGON_MSR_RANGE; else msr &= INTEL_MSR_RANGE; - perf = data->acpi_data; + perf = to_perf_data(data); - for (i = 0; data->freq_table[i].frequency != CPUFREQ_TABLE_END; i++) { - if (msr == perf->states[data->freq_table[i].driver_data].status) - return data->freq_table[i].frequency; - } - return data->freq_table[0].frequency; + cpufreq_for_each_entry(pos, policy->freq_table) + if (msr == perf->states[pos->driver_data].status) + return pos->frequency; + return policy->freq_table[0].frequency; } -static unsigned extract_freq(u32 val, struct acpi_cpufreq_data *data) +static unsigned extract_freq(struct cpufreq_policy *policy, u32 val) { + struct acpi_cpufreq_data *data = policy->driver_data; + switch (data->cpu_feature) { case SYSTEM_INTEL_MSR_CAPABLE: case SYSTEM_AMD_MSR_CAPABLE: - return extract_msr(val, data); + return extract_msr(policy, val); case SYSTEM_IO_CAPABLE: - return extract_io(val, data); + return extract_io(policy, val); default: return 0; } } -struct msr_addr { - u32 reg; -}; +static u32 cpu_freq_read_intel(struct acpi_pct_register *not_used) +{ + u32 val, dummy __always_unused; -struct io_addr { - u16 port; - u8 bit_width; -}; + rdmsr(MSR_IA32_PERF_CTL, val, dummy); + return val; +} + +static void cpu_freq_write_intel(struct acpi_pct_register *not_used, u32 val) +{ + u32 lo, hi; + + rdmsr(MSR_IA32_PERF_CTL, lo, hi); + lo = (lo & ~INTEL_MSR_RANGE) | (val & INTEL_MSR_RANGE); + wrmsr(MSR_IA32_PERF_CTL, lo, hi); +} + +static u32 cpu_freq_read_amd(struct acpi_pct_register *not_used) +{ + u32 val, dummy __always_unused; + + rdmsr(MSR_AMD_PERF_CTL, val, dummy); + return val; +} + +static void cpu_freq_write_amd(struct acpi_pct_register *not_used, u32 val) +{ + wrmsr(MSR_AMD_PERF_CTL, val, 0); +} + +static u32 cpu_freq_read_io(struct acpi_pct_register *reg) +{ + u32 val; + + acpi_os_read_port(reg->address, &val, reg->bit_width); + return val; +} + +static void cpu_freq_write_io(struct acpi_pct_register *reg, u32 val) +{ + acpi_os_write_port(reg->address, val, reg->bit_width); +} struct drv_cmd { - unsigned int type; - const struct cpumask *mask; - union { - struct msr_addr msr; - struct io_addr io; - } addr; + struct acpi_pct_register *reg; u32 val; + union { + void (*write)(struct acpi_pct_register *reg, u32 val); + u32 (*read)(struct acpi_pct_register *reg); + } func; }; /* Called via smp_call_function_single(), on the target CPU */ static void do_drv_read(void *_cmd) { struct drv_cmd *cmd = _cmd; - u32 h; - switch (cmd->type) { - case SYSTEM_INTEL_MSR_CAPABLE: - case SYSTEM_AMD_MSR_CAPABLE: - rdmsr(cmd->addr.msr.reg, cmd->val, h); - break; - case SYSTEM_IO_CAPABLE: - acpi_os_read_port((acpi_io_address)cmd->addr.io.port, - &cmd->val, - (u32)cmd->addr.io.bit_width); - break; - default: - break; - } + cmd->val = cmd->func.read(cmd->reg); } -/* Called via smp_call_function_many(), on the target CPUs */ -static void do_drv_write(void *_cmd) +static u32 drv_read(struct acpi_cpufreq_data *data, const struct cpumask *mask) { - struct drv_cmd *cmd = _cmd; - u32 lo, hi; + struct acpi_processor_performance *perf = to_perf_data(data); + struct drv_cmd cmd = { + .reg = &perf->control_register, + .func.read = data->cpu_freq_read, + }; + int err; - switch (cmd->type) { - case SYSTEM_INTEL_MSR_CAPABLE: - rdmsr(cmd->addr.msr.reg, lo, hi); - lo = (lo & ~INTEL_MSR_RANGE) | (cmd->val & INTEL_MSR_RANGE); - wrmsr(cmd->addr.msr.reg, lo, hi); - break; - case SYSTEM_AMD_MSR_CAPABLE: - wrmsr(cmd->addr.msr.reg, cmd->val, 0); - break; - case SYSTEM_IO_CAPABLE: - acpi_os_write_port((acpi_io_address)cmd->addr.io.port, - cmd->val, - (u32)cmd->addr.io.bit_width); - break; - default: - break; - } + err = smp_call_function_any(mask, do_drv_read, &cmd, 1); + WARN_ON_ONCE(err); /* smp_call_function_any() was buggy? */ + return cmd.val; } -static void drv_read(struct drv_cmd *cmd) +static void do_drv_write(void *_cmd) { - int err; - cmd->val = 0; + struct drv_cmd *cmd = _cmd; - err = smp_call_function_any(cmd->mask, do_drv_read, cmd, 1); - WARN_ON_ONCE(err); /* smp_call_function_any() was buggy? */ + cmd->func.write(cmd->reg, cmd->val); } -static void drv_write(struct drv_cmd *cmd) +static void drv_write(struct acpi_cpufreq_data *data, + const struct cpumask *mask, u32 val) { - int this_cpu; - - this_cpu = get_cpu(); - if (cpumask_test_cpu(this_cpu, cmd->mask)) - do_drv_write(cmd); - smp_call_function_many(cmd->mask, do_drv_write, cmd, 1); - put_cpu(); + struct acpi_processor_performance *perf = to_perf_data(data); + struct drv_cmd cmd = { + .reg = &perf->control_register, + .val = val, + .func.write = data->cpu_freq_write, + }; + + on_each_cpu_mask(mask, do_drv_write, &cmd, true); } -static u32 get_cur_val(const struct cpumask *mask) +static u32 get_cur_val(const struct cpumask *mask, struct acpi_cpufreq_data *data) { - struct acpi_processor_performance *perf; - struct drv_cmd cmd; + u32 val; if (unlikely(cpumask_empty(mask))) return 0; - switch (per_cpu(acfreq_data, cpumask_first(mask))->cpu_feature) { - case SYSTEM_INTEL_MSR_CAPABLE: - cmd.type = SYSTEM_INTEL_MSR_CAPABLE; - cmd.addr.msr.reg = MSR_IA32_PERF_CTL; - break; - case SYSTEM_AMD_MSR_CAPABLE: - cmd.type = SYSTEM_AMD_MSR_CAPABLE; - cmd.addr.msr.reg = MSR_AMD_PERF_CTL; - break; - case SYSTEM_IO_CAPABLE: - cmd.type = SYSTEM_IO_CAPABLE; - perf = per_cpu(acfreq_data, cpumask_first(mask))->acpi_data; - cmd.addr.io.port = perf->control_register.address; - cmd.addr.io.bit_width = perf->control_register.bit_width; - break; - default: - return 0; - } - - cmd.mask = mask; - drv_read(&cmd); + val = drv_read(data, mask); - pr_debug("get_cur_val = %u\n", cmd.val); + pr_debug("%s = %u\n", __func__, val); - return cmd.val; + return val; } static unsigned int get_cur_freq_on_cpu(unsigned int cpu) { - struct acpi_cpufreq_data *data = per_cpu(acfreq_data, cpu); + struct acpi_cpufreq_data *data; + struct cpufreq_policy *policy; unsigned int freq; unsigned int cached_freq; - pr_debug("get_cur_freq_on_cpu (%d)\n", cpu); + pr_debug("%s (%d)\n", __func__, cpu); - if (unlikely(data == NULL || - data->acpi_data == NULL || data->freq_table == NULL)) { + policy = cpufreq_cpu_get_raw(cpu); + if (unlikely(!policy)) + return 0; + + data = policy->driver_data; + if (unlikely(!data || !policy->freq_table)) return 0; - } - cached_freq = data->freq_table[data->acpi_data->state].frequency; - freq = extract_freq(get_cur_val(cpumask_of(cpu)), data); + cached_freq = policy->freq_table[to_perf_data(data)->state].frequency; + freq = extract_freq(policy, get_cur_val(cpumask_of(cpu), data)); if (freq != cached_freq) { /* * The dreaded BIOS frequency change behind our back. @@ -409,50 +384,37 @@ static unsigned int get_cur_freq_on_cpu(unsigned int cpu) return freq; } -static unsigned int check_freqs(const struct cpumask *mask, unsigned int freq, - struct acpi_cpufreq_data *data) +static unsigned int check_freqs(struct cpufreq_policy *policy, + const struct cpumask *mask, unsigned int freq) { + struct acpi_cpufreq_data *data = policy->driver_data; unsigned int cur_freq; unsigned int i; for (i = 0; i < 100; i++) { - cur_freq = extract_freq(get_cur_val(mask), data); + cur_freq = extract_freq(policy, get_cur_val(mask, data)); if (cur_freq == freq) return 1; - udelay(10); + usleep_range(10, 15); } return 0; } static int acpi_cpufreq_target(struct cpufreq_policy *policy, - unsigned int target_freq, unsigned int relation) + unsigned int index) { - struct acpi_cpufreq_data *data = per_cpu(acfreq_data, policy->cpu); + struct acpi_cpufreq_data *data = policy->driver_data; struct acpi_processor_performance *perf; - struct cpufreq_freqs freqs; - struct drv_cmd cmd; - unsigned int next_state = 0; /* Index into freq_table */ + const struct cpumask *mask; unsigned int next_perf_state = 0; /* Index into perf table */ int result = 0; - pr_debug("acpi_cpufreq_target %d (%d)\n", target_freq, policy->cpu); - - if (unlikely(data == NULL || - data->acpi_data == NULL || data->freq_table == NULL)) { + if (unlikely(!data)) { return -ENODEV; } - perf = data->acpi_data; - result = cpufreq_frequency_table_target(policy, - data->freq_table, - target_freq, - relation, &next_state); - if (unlikely(result)) { - result = -ENODEV; - goto out; - } - - next_perf_state = data->freq_table[next_state].driver_data; + perf = to_perf_data(data); + next_perf_state = policy->freq_table[index].driver_data; if (perf->state == next_perf_state) { if (unlikely(data->resume)) { pr_debug("Called after resume, resetting to P%d\n", @@ -461,76 +423,74 @@ static int acpi_cpufreq_target(struct cpufreq_policy *policy, } else { pr_debug("Already at target state (P%d)\n", next_perf_state); - goto out; + return 0; } } - switch (data->cpu_feature) { - case SYSTEM_INTEL_MSR_CAPABLE: - cmd.type = SYSTEM_INTEL_MSR_CAPABLE; - cmd.addr.msr.reg = MSR_IA32_PERF_CTL; - cmd.val = (u32) perf->states[next_perf_state].control; - break; - case SYSTEM_AMD_MSR_CAPABLE: - cmd.type = SYSTEM_AMD_MSR_CAPABLE; - cmd.addr.msr.reg = MSR_AMD_PERF_CTL; - cmd.val = (u32) perf->states[next_perf_state].control; - break; - case SYSTEM_IO_CAPABLE: - cmd.type = SYSTEM_IO_CAPABLE; - cmd.addr.io.port = perf->control_register.address; - cmd.addr.io.bit_width = perf->control_register.bit_width; - cmd.val = (u32) perf->states[next_perf_state].control; - break; - default: - result = -ENODEV; - goto out; - } - - /* cpufreq holds the hotplug lock, so we are safe from here on */ - if (policy->shared_type != CPUFREQ_SHARED_TYPE_ANY) - cmd.mask = policy->cpus; - else - cmd.mask = cpumask_of(policy->cpu); - - freqs.old = perf->states[perf->state].core_frequency * 1000; - freqs.new = data->freq_table[next_state].frequency; - cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE); + /* + * The core won't allow CPUs to go away until the governor has been + * stopped, so we can rely on the stability of policy->cpus. + */ + mask = policy->shared_type == CPUFREQ_SHARED_TYPE_ANY ? + cpumask_of(policy->cpu) : policy->cpus; - drv_write(&cmd); + drv_write(data, mask, perf->states[next_perf_state].control); if (acpi_pstate_strict) { - if (!check_freqs(cmd.mask, freqs.new, data)) { - pr_debug("acpi_cpufreq_target failed (%d)\n", - policy->cpu); + if (!check_freqs(policy, mask, + policy->freq_table[index].frequency)) { + pr_debug("%s (%d)\n", __func__, policy->cpu); result = -EAGAIN; - freqs.new = freqs.old; } } - cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); - if (!result) perf->state = next_perf_state; -out: return result; } -static int acpi_cpufreq_verify(struct cpufreq_policy *policy) +static unsigned int acpi_cpufreq_fast_switch(struct cpufreq_policy *policy, + unsigned int target_freq) { - struct acpi_cpufreq_data *data = per_cpu(acfreq_data, policy->cpu); + struct acpi_cpufreq_data *data = policy->driver_data; + struct acpi_processor_performance *perf; + struct cpufreq_frequency_table *entry; + unsigned int next_perf_state, next_freq, index; + + /* + * Find the closest frequency above target_freq. + */ + if (policy->cached_target_freq == target_freq) + index = policy->cached_resolved_idx; + else + index = cpufreq_table_find_index_dl(policy, target_freq, + false); + + entry = &policy->freq_table[index]; + next_freq = entry->frequency; + next_perf_state = entry->driver_data; - pr_debug("acpi_cpufreq_verify\n"); + perf = to_perf_data(data); + if (perf->state == next_perf_state) { + if (unlikely(data->resume)) + data->resume = 0; + else + return next_freq; + } - return cpufreq_frequency_table_verify(policy, data->freq_table); + data->cpu_freq_write(&perf->control_register, + perf->states[next_perf_state].control); + perf->state = next_perf_state; + return next_freq; } static unsigned long acpi_cpufreq_guess_freq(struct acpi_cpufreq_data *data, unsigned int cpu) { - struct acpi_processor_performance *perf = data->acpi_data; + struct acpi_processor_performance *perf; + perf = to_perf_data(data); if (cpu_khz) { /* search the closest match to cpu_khz */ unsigned int i; @@ -565,44 +525,15 @@ static void free_acpi_perf_data(void) free_percpu(acpi_perf_data); } -static int boost_notify(struct notifier_block *nb, unsigned long action, - void *hcpu) +static int cpufreq_boost_down_prep(unsigned int cpu) { - unsigned cpu = (long)hcpu; - const struct cpumask *cpumask; - - cpumask = get_cpu_mask(cpu); - /* * Clear the boost-disable bit on the CPU_DOWN path so that - * this cpu cannot block the remaining ones from boosting. On - * the CPU_UP path we simply keep the boost-disable flag in - * sync with the current global state. + * this cpu cannot block the remaining ones from boosting. */ - - switch (action) { - case CPU_UP_PREPARE: - case CPU_UP_PREPARE_FROZEN: - boost_set_msrs(boost_enabled, cpumask); - break; - - case CPU_DOWN_PREPARE: - case CPU_DOWN_PREPARE_FROZEN: - boost_set_msrs(1, cpumask); - break; - - default: - break; - } - - return NOTIFY_OK; + return boost_set_msr(1); } - -static struct notifier_block boost_nb = { - .notifier_call = boost_notify, -}; - /* * acpi_cpufreq_early_init - initialize ACPI P-States library * @@ -614,7 +545,7 @@ static struct notifier_block boost_nb = { static int __init acpi_cpufreq_early_init(void) { unsigned int i; - pr_debug("acpi_cpufreq_early_init\n"); + pr_debug("%s\n", __func__); acpi_perf_data = alloc_percpu(struct acpi_processor_performance); if (!acpi_perf_data) { @@ -668,18 +599,15 @@ static const struct dmi_system_id sw_any_bug_dmi_table[] = { static int acpi_cpufreq_blacklist(struct cpuinfo_x86 *c) { /* Intel Xeon Processor 7100 Series Specification Update - * http://www.intel.com/Assets/PDF/specupdate/314554.pdf + * https://www.intel.com/Assets/PDF/specupdate/314554.pdf * AL30: A Machine Check Exception (MCE) Occurring during an * Enhanced Intel SpeedStep Technology Ratio Change May Cause * Both Processor Cores to Lock Up. */ if (c->x86_vendor == X86_VENDOR_INTEL) { if ((c->x86 == 15) && (c->x86_model == 6) && - (c->x86_mask == 8)) { - printk(KERN_INFO "acpi-cpufreq: Intel(R) " - "Xeon(R) 7100 Errata AL30, processors may " - "lock up on frequency changes: disabling " - "acpi-cpufreq.\n"); + (c->x86_stepping == 8)) { + pr_info("Intel(R) Xeon(R) 7100 Errata AL30, processors may lock up on frequency changes: disabling acpi-cpufreq\n"); return -ENODEV; } } @@ -687,20 +615,82 @@ static int acpi_cpufreq_blacklist(struct cpuinfo_x86 *c) } #endif +#ifdef CONFIG_ACPI_CPPC_LIB +/* + * get_max_boost_ratio: Computes the max_boost_ratio as the ratio + * between the highest_perf and the nominal_perf. + * + * Returns the max_boost_ratio for @cpu. Returns the CPPC nominal + * frequency via @nominal_freq if it is non-NULL pointer. + */ +static u64 get_max_boost_ratio(unsigned int cpu, u64 *nominal_freq) +{ + struct cppc_perf_caps perf_caps; + u64 highest_perf, nominal_perf; + int ret; + + if (acpi_pstate_strict) + return 0; + + ret = cppc_get_perf_caps(cpu, &perf_caps); + if (ret) { + pr_debug("CPU%d: Unable to get performance capabilities (%d)\n", + cpu, ret); + return 0; + } + + if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) { + ret = amd_get_boost_ratio_numerator(cpu, &highest_perf); + if (ret) { + pr_debug("CPU%d: Unable to get boost ratio numerator (%d)\n", + cpu, ret); + return 0; + } + } else { + highest_perf = perf_caps.highest_perf; + } + + nominal_perf = perf_caps.nominal_perf; + + if (nominal_freq) + *nominal_freq = perf_caps.nominal_freq * 1000; + + if (!highest_perf || !nominal_perf) { + pr_debug("CPU%d: highest or nominal performance missing\n", cpu); + return 0; + } + + if (highest_perf < nominal_perf) { + pr_debug("CPU%d: nominal performance above highest\n", cpu); + return 0; + } + + return div_u64(highest_perf << SCHED_CAPACITY_SHIFT, nominal_perf); +} + +#else +static inline u64 get_max_boost_ratio(unsigned int cpu, u64 *nominal_freq) +{ + return 0; +} +#endif + static int acpi_cpufreq_cpu_init(struct cpufreq_policy *policy) { - unsigned int i; - unsigned int valid_states = 0; - unsigned int cpu = policy->cpu; + struct cpufreq_frequency_table *freq_table; + struct acpi_processor_performance *perf; struct acpi_cpufreq_data *data; + unsigned int cpu = policy->cpu; + struct cpuinfo_x86 *c = &cpu_data(cpu); + u64 max_boost_ratio, nominal_freq = 0; + unsigned int valid_states = 0; unsigned int result = 0; - struct cpuinfo_x86 *c = &cpu_data(policy->cpu); - struct acpi_processor_performance *perf; + unsigned int i; #ifdef CONFIG_SMP static int blacklisted; #endif - pr_debug("acpi_cpufreq_cpu_init\n"); + pr_debug("%s\n", __func__); #ifdef CONFIG_SMP if (blacklisted) @@ -710,7 +700,7 @@ static int acpi_cpufreq_cpu_init(struct cpufreq_policy *policy) return blacklisted; #endif - data = kzalloc(sizeof(struct acpi_cpufreq_data), GFP_KERNEL); + data = kzalloc(sizeof(*data), GFP_KERNEL); if (!data) return -ENOMEM; @@ -719,17 +709,17 @@ static int acpi_cpufreq_cpu_init(struct cpufreq_policy *policy) goto err_free; } - data->acpi_data = per_cpu_ptr(acpi_perf_data, cpu); - per_cpu(acfreq_data, cpu) = data; + perf = per_cpu_ptr(acpi_perf_data, cpu); + data->acpi_perf_cpu = cpu; + policy->driver_data = data; if (cpu_has(c, X86_FEATURE_CONSTANT_TSC)) acpi_cpufreq_driver.flags |= CPUFREQ_CONST_LOOPS; - result = acpi_processor_register_performance(data->acpi_data, cpu); + result = acpi_processor_register_performance(perf, cpu); if (result) goto err_free_mask; - perf = data->acpi_data; policy->shared_type = perf->shared_type; /* @@ -746,15 +736,17 @@ static int acpi_cpufreq_cpu_init(struct cpufreq_policy *policy) dmi_check_system(sw_any_bug_dmi_table); if (bios_with_sw_any_bug && !policy_is_shared(policy)) { policy->shared_type = CPUFREQ_SHARED_TYPE_ALL; - cpumask_copy(policy->cpus, cpu_core_mask(cpu)); + cpumask_copy(policy->cpus, topology_core_cpumask(cpu)); } - if (check_amd_hwpstate_cpu(cpu) && !acpi_pstate_strict) { + if (check_amd_hwpstate_cpu(cpu) && boot_cpu_data.x86 < 0x19 && + !acpi_pstate_strict) { cpumask_clear(policy->cpus); cpumask_set_cpu(cpu, policy->cpus); - cpumask_copy(data->freqdomain_cpus, cpu_sibling_mask(cpu)); + cpumask_copy(data->freqdomain_cpus, + topology_sibling_cpumask(cpu)); policy->shared_type = CPUFREQ_SHARED_TYPE_HW; - pr_info_once(PFX "overriding BIOS provided _PSD data\n"); + pr_info_once("overriding BIOS provided _PSD data\n"); } #endif @@ -780,15 +772,21 @@ static int acpi_cpufreq_cpu_init(struct cpufreq_policy *policy) } pr_debug("SYSTEM IO addr space\n"); data->cpu_feature = SYSTEM_IO_CAPABLE; + data->cpu_freq_read = cpu_freq_read_io; + data->cpu_freq_write = cpu_freq_write_io; break; case ACPI_ADR_SPACE_FIXED_HARDWARE: pr_debug("HARDWARE addr space\n"); if (check_est_cpu(cpu)) { data->cpu_feature = SYSTEM_INTEL_MSR_CAPABLE; + data->cpu_freq_read = cpu_freq_read_intel; + data->cpu_freq_write = cpu_freq_write_intel; break; } if (check_amd_hwpstate_cpu(cpu)) { data->cpu_feature = SYSTEM_AMD_MSR_CAPABLE; + data->cpu_freq_read = cpu_freq_read_amd; + data->cpu_freq_write = cpu_freq_write_amd; break; } result = -ENODEV; @@ -800,9 +798,9 @@ static int acpi_cpufreq_cpu_init(struct cpufreq_policy *policy) goto err_unreg; } - data->freq_table = kmalloc(sizeof(struct cpufreq_frequency_table) * - (perf->state_count+1), GFP_KERNEL); - if (!data->freq_table) { + freq_table = kcalloc(perf->state_count + 1, sizeof(*freq_table), + GFP_KERNEL); + if (!freq_table) { result = -ENOMEM; goto err_unreg; } @@ -820,39 +818,62 @@ static int acpi_cpufreq_cpu_init(struct cpufreq_policy *policy) if (perf->control_register.space_id == ACPI_ADR_SPACE_FIXED_HARDWARE && policy->cpuinfo.transition_latency > 20 * 1000) { policy->cpuinfo.transition_latency = 20 * 1000; - printk_once(KERN_INFO - "P-state transition latency capped at 20 uS\n"); + pr_info_once("P-state transition latency capped at 20 uS\n"); } /* table init */ for (i = 0; i < perf->state_count; i++) { if (i > 0 && perf->states[i].core_frequency >= - data->freq_table[valid_states-1].frequency / 1000) + freq_table[valid_states-1].frequency / 1000) continue; - data->freq_table[valid_states].driver_data = i; - data->freq_table[valid_states].frequency = + freq_table[valid_states].driver_data = i; + freq_table[valid_states].frequency = perf->states[i].core_frequency * 1000; valid_states++; } - data->freq_table[valid_states].frequency = CPUFREQ_TABLE_END; - perf->state = 0; + freq_table[valid_states].frequency = CPUFREQ_TABLE_END; - result = cpufreq_frequency_table_cpuinfo(policy, data->freq_table); - if (result) - goto err_freqfree; + max_boost_ratio = get_max_boost_ratio(cpu, &nominal_freq); + if (max_boost_ratio) { + unsigned int freq = nominal_freq; - if (perf->states[0].core_frequency * 1000 != policy->cpuinfo.max_freq) - printk(KERN_WARNING FW_WARN "P-state 0 is not max freq\n"); + /* + * The loop above sorts the freq_table entries in the + * descending order. If ACPI CPPC has not advertised + * the nominal frequency (this is possible in CPPC + * revisions prior to 3), then use the first entry in + * the pstate table as a proxy for nominal frequency. + */ + if (!freq) + freq = freq_table[0].frequency; + + policy->cpuinfo.max_freq = freq * max_boost_ratio >> SCHED_CAPACITY_SHIFT; + } else { + /* + * If the maximum "boost" frequency is unknown, ask the arch + * scale-invariance code to use the "nominal" performance for + * CPU utilization scaling so as to prevent the schedutil + * governor from selecting inadequate CPU frequencies. + */ + arch_set_max_freq_ratio(true); + } + + policy->freq_table = freq_table; + perf->state = 0; switch (perf->control_register.space_id) { case ACPI_ADR_SPACE_SYSTEM_IO: - /* Current speed is unknown and not detectable by IO port */ + /* + * The core will not set policy->cur, because + * cpufreq_driver->get is NULL, so we need to set it here. + * However, we have to guess it, because the current speed is + * unknown and not detectable via IO ports. + */ policy->cur = acpi_cpufreq_guess_freq(data, policy->cpu); break; case ACPI_ADR_SPACE_FIXED_HARDWARE: acpi_cpufreq_driver.get = get_cur_freq_on_cpu; - policy->cur = get_cur_freq_on_cpu(cpu); break; default: break; @@ -861,10 +882,6 @@ static int acpi_cpufreq_cpu_init(struct cpufreq_policy *policy) /* notify BIOS that we exist */ acpi_processor_notify_smm(THIS_MODULE); - /* Check for APERF/MPERF support in hardware */ - if (boot_cpu_has(X86_FEATURE_APERFMPERF)) - acpi_cpufreq_driver.getavg = cpufreq_get_measured_perf; - pr_debug("CPU%u - ACPI performance management activated.\n", cpu); for (i = 0; i < perf->state_count; i++) pr_debug(" %cP%d: %d MHz, %d mW, %d uS\n", @@ -873,53 +890,65 @@ static int acpi_cpufreq_cpu_init(struct cpufreq_policy *policy) (u32) perf->states[i].power, (u32) perf->states[i].transition_latency); - cpufreq_frequency_table_get_attr(data->freq_table, policy->cpu); - /* * the first call to ->target() should result in us actually * writing something to the appropriate registers. */ data->resume = 1; + policy->fast_switch_possible = !acpi_pstate_strict && + !(policy_is_shared(policy) && policy->shared_type != CPUFREQ_SHARED_TYPE_ANY); + + if (perf->states[0].core_frequency * 1000 != freq_table[0].frequency) + pr_warn(FW_WARN "P-state 0 is not max freq\n"); + + if (acpi_cpufreq_driver.set_boost) { + if (policy->boost_supported) { + /* + * The firmware may have altered boost state while the + * CPU was offline (for example during a suspend-resume + * cycle). + */ + if (policy->boost_enabled != boost_state(cpu)) + set_boost(policy, policy->boost_enabled); + } else { + policy->boost_supported = true; + } + } + return result; -err_freqfree: - kfree(data->freq_table); err_unreg: - acpi_processor_unregister_performance(perf, cpu); + acpi_processor_unregister_performance(cpu); err_free_mask: free_cpumask_var(data->freqdomain_cpus); err_free: kfree(data); - per_cpu(acfreq_data, cpu) = NULL; + policy->driver_data = NULL; return result; } -static int acpi_cpufreq_cpu_exit(struct cpufreq_policy *policy) +static void acpi_cpufreq_cpu_exit(struct cpufreq_policy *policy) { - struct acpi_cpufreq_data *data = per_cpu(acfreq_data, policy->cpu); - - pr_debug("acpi_cpufreq_cpu_exit\n"); - - if (data) { - cpufreq_frequency_table_put_attr(policy->cpu); - per_cpu(acfreq_data, policy->cpu) = NULL; - acpi_processor_unregister_performance(data->acpi_data, - policy->cpu); - free_cpumask_var(data->freqdomain_cpus); - kfree(data->freq_table); - kfree(data); - } + struct acpi_cpufreq_data *data = policy->driver_data; - return 0; + pr_debug("%s\n", __func__); + + cpufreq_boost_down_prep(policy->cpu); + policy->fast_switch_possible = false; + policy->driver_data = NULL; + acpi_processor_unregister_performance(data->acpi_perf_cpu); + free_cpumask_var(data->freqdomain_cpus); + kfree(policy->freq_table); + kfree(data); } static int acpi_cpufreq_resume(struct cpufreq_policy *policy) { - struct acpi_cpufreq_data *data = per_cpu(acfreq_data, policy->cpu); + struct acpi_cpufreq_data *data = policy->driver_data; - pr_debug("acpi_cpufreq_resume\n"); + pr_debug("%s\n", __func__); data->resume = 1; @@ -927,75 +956,48 @@ static int acpi_cpufreq_resume(struct cpufreq_policy *policy) } static struct freq_attr *acpi_cpufreq_attr[] = { - &cpufreq_freq_attr_scaling_available_freqs, &freqdomain_cpus, - NULL, /* this is a placeholder for cpb, do not remove */ +#ifdef CONFIG_X86_ACPI_CPUFREQ_CPB + &cpb, +#endif NULL, }; static struct cpufreq_driver acpi_cpufreq_driver = { - .verify = acpi_cpufreq_verify, - .target = acpi_cpufreq_target, + .verify = cpufreq_generic_frequency_table_verify, + .target_index = acpi_cpufreq_target, + .fast_switch = acpi_cpufreq_fast_switch, .bios_limit = acpi_processor_get_bios_limit, .init = acpi_cpufreq_cpu_init, .exit = acpi_cpufreq_cpu_exit, .resume = acpi_cpufreq_resume, .name = "acpi-cpufreq", - .owner = THIS_MODULE, .attr = acpi_cpufreq_attr, }; static void __init acpi_cpufreq_boost_init(void) { - if (boot_cpu_has(X86_FEATURE_CPB) || boot_cpu_has(X86_FEATURE_IDA)) { - msrs = msrs_alloc(); - - if (!msrs) - return; - - boost_supported = true; - boost_enabled = boost_state(0); - - get_online_cpus(); - - /* Force all MSRs to the same value */ - boost_set_msrs(boost_enabled, cpu_online_mask); - - register_cpu_notifier(&boost_nb); - - put_online_cpus(); - } else - global_boost.attr.mode = 0444; - - /* We create the boost file in any case, though for systems without - * hardware support it will be read-only and hardwired to return 0. - */ - if (cpufreq_sysfs_create_file(&(global_boost.attr))) - pr_warn(PFX "could not register global boost sysfs file\n"); - else - pr_debug("registered global boost sysfs file\n"); -} - -static void __exit acpi_cpufreq_boost_exit(void) -{ - cpufreq_sysfs_remove_file(&(global_boost.attr)); - - if (msrs) { - unregister_cpu_notifier(&boost_nb); - - msrs_free(msrs); - msrs = NULL; + if (!(boot_cpu_has(X86_FEATURE_CPB) || boot_cpu_has(X86_FEATURE_IDA))) { + pr_debug("Boost capabilities not present in the processor\n"); + return; } + + acpi_cpufreq_driver.set_boost = set_boost; + acpi_cpufreq_driver.boost_enabled = boost_state(0); } -static int __init acpi_cpufreq_init(void) +static int __init acpi_cpufreq_probe(struct platform_device *pdev) { int ret; if (acpi_disabled) - return 0; + return -ENODEV; - pr_debug("acpi_cpufreq_init\n"); + /* don't keep reloading if cpufreq_driver exists */ + if (cpufreq_get_current_driver()) + return -ENODEV; + + pr_debug("%s\n", __func__); ret = acpi_cpufreq_early_init(); if (ret) @@ -1008,40 +1010,53 @@ static int __init acpi_cpufreq_init(void) * only if configured. This is considered legacy code, which * will probably be removed at some point in the future. */ - if (check_amd_hwpstate_cpu(0)) { - struct freq_attr **iter; + if (!check_amd_hwpstate_cpu(0)) { + struct freq_attr **attr; - pr_debug("adding sysfs entry for cpb\n"); + pr_debug("CPB unsupported, do not expose it\n"); - for (iter = acpi_cpufreq_attr; *iter != NULL; iter++) - ; - - /* make sure there is a terminator behind it */ - if (iter[1] == NULL) - *iter = &cpb; + for (attr = acpi_cpufreq_attr; *attr; attr++) + if (*attr == &cpb) { + *attr = NULL; + break; + } } #endif + acpi_cpufreq_boost_init(); ret = cpufreq_register_driver(&acpi_cpufreq_driver); - if (ret) + if (ret) { free_acpi_perf_data(); - else - acpi_cpufreq_boost_init(); - + } return ret; } -static void __exit acpi_cpufreq_exit(void) +static void acpi_cpufreq_remove(struct platform_device *pdev) { - pr_debug("acpi_cpufreq_exit\n"); - - acpi_cpufreq_boost_exit(); + pr_debug("%s\n", __func__); cpufreq_unregister_driver(&acpi_cpufreq_driver); free_acpi_perf_data(); } +static struct platform_driver acpi_cpufreq_platdrv = { + .driver = { + .name = "acpi-cpufreq", + }, + .remove = acpi_cpufreq_remove, +}; + +static int __init acpi_cpufreq_init(void) +{ + return platform_driver_probe(&acpi_cpufreq_platdrv, acpi_cpufreq_probe); +} + +static void __exit acpi_cpufreq_exit(void) +{ + platform_driver_unregister(&acpi_cpufreq_platdrv); +} + module_param(acpi_pstate_strict, uint, 0644); MODULE_PARM_DESC(acpi_pstate_strict, "value 0 or non-zero. non-zero -> strict ACPI checks are " @@ -1050,18 +1065,4 @@ MODULE_PARM_DESC(acpi_pstate_strict, late_initcall(acpi_cpufreq_init); module_exit(acpi_cpufreq_exit); -static const struct x86_cpu_id acpi_cpufreq_ids[] = { - X86_FEATURE_MATCH(X86_FEATURE_ACPI), - X86_FEATURE_MATCH(X86_FEATURE_HW_PSTATE), - {} -}; -MODULE_DEVICE_TABLE(x86cpu, acpi_cpufreq_ids); - -static const struct acpi_device_id processor_device_ids[] = { - {ACPI_PROCESSOR_OBJECT_HID, }, - {ACPI_PROCESSOR_DEVICE_HID, }, - {}, -}; -MODULE_DEVICE_TABLE(acpi, processor_device_ids); - -MODULE_ALIAS("acpi"); +MODULE_ALIAS("platform:acpi-cpufreq"); diff --git a/drivers/cpufreq/airoha-cpufreq.c b/drivers/cpufreq/airoha-cpufreq.c new file mode 100644 index 000000000000..b6b1cdc4d11d --- /dev/null +++ b/drivers/cpufreq/airoha-cpufreq.c @@ -0,0 +1,153 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/bitfield.h> +#include <linux/cpufreq.h> +#include <linux/module.h> +#include <linux/platform_device.h> +#include <linux/pm_domain.h> +#include <linux/pm_runtime.h> +#include <linux/slab.h> + +#include "cpufreq-dt.h" + +struct airoha_cpufreq_priv { + int opp_token; + struct dev_pm_domain_list *pd_list; + struct platform_device *cpufreq_dt; +}; + +static struct platform_device *cpufreq_pdev; + +/* NOP function to disable OPP from setting clock */ +static int airoha_cpufreq_config_clks_nop(struct device *dev, + struct opp_table *opp_table, + struct dev_pm_opp *opp, + void *data, bool scaling_down) +{ + return 0; +} + +static const char * const airoha_cpufreq_clk_names[] = { "cpu", NULL }; +static const char * const airoha_cpufreq_pd_names[] = { "perf" }; + +static int airoha_cpufreq_probe(struct platform_device *pdev) +{ + const struct dev_pm_domain_attach_data attach_data = { + .pd_names = airoha_cpufreq_pd_names, + .num_pd_names = ARRAY_SIZE(airoha_cpufreq_pd_names), + .pd_flags = PD_FLAG_DEV_LINK_ON | PD_FLAG_REQUIRED_OPP, + }; + struct dev_pm_opp_config config = { + .clk_names = airoha_cpufreq_clk_names, + .config_clks = airoha_cpufreq_config_clks_nop, + }; + struct platform_device *cpufreq_dt; + struct airoha_cpufreq_priv *priv; + struct device *dev = &pdev->dev; + struct device *cpu_dev; + int ret; + + /* CPUs refer to the same OPP table */ + cpu_dev = get_cpu_device(0); + if (!cpu_dev) + return -ENODEV; + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + /* Set OPP table conf with NOP config_clks */ + priv->opp_token = dev_pm_opp_set_config(cpu_dev, &config); + if (priv->opp_token < 0) + return dev_err_probe(dev, priv->opp_token, "Failed to set OPP config\n"); + + /* Attach PM for OPP */ + ret = dev_pm_domain_attach_list(cpu_dev, &attach_data, + &priv->pd_list); + if (ret) + goto clear_opp_config; + + cpufreq_dt = platform_device_register_simple("cpufreq-dt", -1, NULL, 0); + ret = PTR_ERR_OR_ZERO(cpufreq_dt); + if (ret) { + dev_err(dev, "failed to create cpufreq-dt device: %d\n", ret); + goto detach_pm; + } + + priv->cpufreq_dt = cpufreq_dt; + platform_set_drvdata(pdev, priv); + + return 0; + +detach_pm: + dev_pm_domain_detach_list(priv->pd_list); +clear_opp_config: + dev_pm_opp_clear_config(priv->opp_token); + + return ret; +} + +static void airoha_cpufreq_remove(struct platform_device *pdev) +{ + struct airoha_cpufreq_priv *priv = platform_get_drvdata(pdev); + + platform_device_unregister(priv->cpufreq_dt); + + dev_pm_domain_detach_list(priv->pd_list); + + dev_pm_opp_clear_config(priv->opp_token); +} + +static struct platform_driver airoha_cpufreq_driver = { + .probe = airoha_cpufreq_probe, + .remove = airoha_cpufreq_remove, + .driver = { + .name = "airoha-cpufreq", + }, +}; + +static const struct of_device_id airoha_cpufreq_match_list[] __initconst = { + { .compatible = "airoha,an7583" }, + { .compatible = "airoha,en7581" }, + {}, +}; +MODULE_DEVICE_TABLE(of, airoha_cpufreq_match_list); + +static int __init airoha_cpufreq_init(void) +{ + struct device_node *np = of_find_node_by_path("/"); + const struct of_device_id *match; + int ret; + + if (!np) + return -ENODEV; + + match = of_match_node(airoha_cpufreq_match_list, np); + of_node_put(np); + if (!match) + return -ENODEV; + + ret = platform_driver_register(&airoha_cpufreq_driver); + if (unlikely(ret < 0)) + return ret; + + cpufreq_pdev = platform_device_register_data(NULL, "airoha-cpufreq", + -1, match, sizeof(*match)); + ret = PTR_ERR_OR_ZERO(cpufreq_pdev); + if (ret) + platform_driver_unregister(&airoha_cpufreq_driver); + + return ret; +} +module_init(airoha_cpufreq_init); + +static void __exit airoha_cpufreq_exit(void) +{ + platform_device_unregister(cpufreq_pdev); + platform_driver_unregister(&airoha_cpufreq_driver); +} +module_exit(airoha_cpufreq_exit); + +MODULE_AUTHOR("Christian Marangi <ansuelsmth@gmail.com>"); +MODULE_DESCRIPTION("CPUfreq driver for Airoha SoCs"); +MODULE_LICENSE("GPL"); diff --git a/drivers/cpufreq/amd-pstate-trace.c b/drivers/cpufreq/amd-pstate-trace.c new file mode 100644 index 000000000000..891b696dcd69 --- /dev/null +++ b/drivers/cpufreq/amd-pstate-trace.c @@ -0,0 +1,2 @@ +#define CREATE_TRACE_POINTS +#include "amd-pstate-trace.h" diff --git a/drivers/cpufreq/amd-pstate-trace.h b/drivers/cpufreq/amd-pstate-trace.h new file mode 100644 index 000000000000..32e1bdc588c5 --- /dev/null +++ b/drivers/cpufreq/amd-pstate-trace.h @@ -0,0 +1,142 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * amd-pstate-trace.h - AMD Processor P-state Frequency Driver Tracer + * + * Copyright (C) 2021 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Author: Huang Rui <ray.huang@amd.com> + */ + +#if !defined(_AMD_PSTATE_TRACE_H) || defined(TRACE_HEADER_MULTI_READ) +#define _AMD_PSTATE_TRACE_H + +#include <linux/cpufreq.h> +#include <linux/tracepoint.h> +#include <linux/trace_events.h> + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM amd_cpu + +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE amd-pstate-trace + +#define TPS(x) tracepoint_string(x) + +TRACE_EVENT(amd_pstate_perf, + + TP_PROTO(u8 min_perf, + u8 target_perf, + u8 capacity, + u64 freq, + u64 mperf, + u64 aperf, + u64 tsc, + unsigned int cpu_id, + bool fast_switch + ), + + TP_ARGS(min_perf, + target_perf, + capacity, + freq, + mperf, + aperf, + tsc, + cpu_id, + fast_switch + ), + + TP_STRUCT__entry( + __field(u8, min_perf) + __field(u8, target_perf) + __field(u8, capacity) + __field(unsigned long long, freq) + __field(unsigned long long, mperf) + __field(unsigned long long, aperf) + __field(unsigned long long, tsc) + __field(unsigned int, cpu_id) + __field(bool, fast_switch) + ), + + TP_fast_assign( + __entry->min_perf = min_perf; + __entry->target_perf = target_perf; + __entry->capacity = capacity; + __entry->freq = freq; + __entry->mperf = mperf; + __entry->aperf = aperf; + __entry->tsc = tsc; + __entry->cpu_id = cpu_id; + __entry->fast_switch = fast_switch; + ), + + TP_printk("amd_min_perf=%hhu amd_des_perf=%hhu amd_max_perf=%hhu freq=%llu mperf=%llu aperf=%llu tsc=%llu cpu_id=%u fast_switch=%s", + (u8)__entry->min_perf, + (u8)__entry->target_perf, + (u8)__entry->capacity, + (unsigned long long)__entry->freq, + (unsigned long long)__entry->mperf, + (unsigned long long)__entry->aperf, + (unsigned long long)__entry->tsc, + (unsigned int)__entry->cpu_id, + (__entry->fast_switch) ? "true" : "false" + ) +); + +TRACE_EVENT(amd_pstate_epp_perf, + + TP_PROTO(unsigned int cpu_id, + u8 highest_perf, + u8 epp, + u8 min_perf, + u8 max_perf, + bool boost, + bool changed + ), + + TP_ARGS(cpu_id, + highest_perf, + epp, + min_perf, + max_perf, + boost, + changed), + + TP_STRUCT__entry( + __field(unsigned int, cpu_id) + __field(u8, highest_perf) + __field(u8, epp) + __field(u8, min_perf) + __field(u8, max_perf) + __field(bool, boost) + __field(bool, changed) + ), + + TP_fast_assign( + __entry->cpu_id = cpu_id; + __entry->highest_perf = highest_perf; + __entry->epp = epp; + __entry->min_perf = min_perf; + __entry->max_perf = max_perf; + __entry->boost = boost; + __entry->changed = changed; + ), + + TP_printk("cpu%u: [%hhu<->%hhu]/%hhu, epp=%hhu, boost=%u, changed=%u", + (unsigned int)__entry->cpu_id, + (u8)__entry->min_perf, + (u8)__entry->max_perf, + (u8)__entry->highest_perf, + (u8)__entry->epp, + (bool)__entry->boost, + (bool)__entry->changed + ) +); + +#endif /* _AMD_PSTATE_TRACE_H */ + +/* This part must be outside protection */ +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . + +#include <trace/define_trace.h> diff --git a/drivers/cpufreq/amd-pstate-ut.c b/drivers/cpufreq/amd-pstate-ut.c new file mode 100644 index 000000000000..447b9aa5ce40 --- /dev/null +++ b/drivers/cpufreq/amd-pstate-ut.c @@ -0,0 +1,298 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * AMD Processor P-state Frequency Driver Unit Test + * + * Copyright (C) 2022 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Author: Meng Li <li.meng@amd.com> + * + * The AMD P-State Unit Test is a test module for testing the amd-pstate + * driver. 1) It can help all users to verify their processor support + * (SBIOS/Firmware or Hardware). 2) Kernel can have a basic function + * test to avoid the kernel regression during the update. 3) We can + * introduce more functional or performance tests to align the result + * together, it will benefit power and performance scale optimization. + * + * This driver implements basic framework with plans to enhance it with + * additional test cases to improve the depth and coverage of the test. + * + * See Documentation/admin-guide/pm/amd-pstate.rst Unit Tests for + * amd-pstate to get more detail. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/bitfield.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/moduleparam.h> +#include <linux/fs.h> +#include <linux/cleanup.h> + +#include <acpi/cppc_acpi.h> + +#include <asm/msr.h> + +#include "amd-pstate.h" + + +struct amd_pstate_ut_struct { + const char *name; + int (*func)(u32 index); +}; + +/* + * Kernel module for testing the AMD P-State unit test + */ +static int amd_pstate_ut_acpi_cpc_valid(u32 index); +static int amd_pstate_ut_check_enabled(u32 index); +static int amd_pstate_ut_check_perf(u32 index); +static int amd_pstate_ut_check_freq(u32 index); +static int amd_pstate_ut_check_driver(u32 index); + +static struct amd_pstate_ut_struct amd_pstate_ut_cases[] = { + {"amd_pstate_ut_acpi_cpc_valid", amd_pstate_ut_acpi_cpc_valid }, + {"amd_pstate_ut_check_enabled", amd_pstate_ut_check_enabled }, + {"amd_pstate_ut_check_perf", amd_pstate_ut_check_perf }, + {"amd_pstate_ut_check_freq", amd_pstate_ut_check_freq }, + {"amd_pstate_ut_check_driver", amd_pstate_ut_check_driver } +}; + +static bool get_shared_mem(void) +{ + bool result = false; + + if (!boot_cpu_has(X86_FEATURE_CPPC)) + result = true; + + return result; +} + +/* + * check the _CPC object is present in SBIOS. + */ +static int amd_pstate_ut_acpi_cpc_valid(u32 index) +{ + if (!acpi_cpc_valid()) { + pr_err("%s the _CPC object is not present in SBIOS!\n", __func__); + return -EINVAL; + } + + return 0; +} + +/* + * check if amd pstate is enabled + */ +static int amd_pstate_ut_check_enabled(u32 index) +{ + u64 cppc_enable = 0; + int ret; + + if (get_shared_mem()) + return 0; + + ret = rdmsrq_safe(MSR_AMD_CPPC_ENABLE, &cppc_enable); + if (ret) { + pr_err("%s rdmsrq_safe MSR_AMD_CPPC_ENABLE ret=%d error!\n", __func__, ret); + return ret; + } + + if (!cppc_enable) { + pr_err("%s amd pstate must be enabled!\n", __func__); + return -EINVAL; + } + + return 0; +} + +/* + * check if performance values are reasonable. + * highest_perf >= nominal_perf > lowest_nonlinear_perf > lowest_perf > 0 + */ +static int amd_pstate_ut_check_perf(u32 index) +{ + int cpu = 0, ret = 0; + u32 highest_perf = 0, nominal_perf = 0, lowest_nonlinear_perf = 0, lowest_perf = 0; + u64 cap1 = 0; + struct cppc_perf_caps cppc_perf; + union perf_cached cur_perf; + + for_each_online_cpu(cpu) { + struct cpufreq_policy *policy __free(put_cpufreq_policy) = NULL; + struct amd_cpudata *cpudata; + + policy = cpufreq_cpu_get(cpu); + if (!policy) + continue; + cpudata = policy->driver_data; + + if (get_shared_mem()) { + ret = cppc_get_perf_caps(cpu, &cppc_perf); + if (ret) { + pr_err("%s cppc_get_perf_caps ret=%d error!\n", __func__, ret); + return ret; + } + + highest_perf = cppc_perf.highest_perf; + nominal_perf = cppc_perf.nominal_perf; + lowest_nonlinear_perf = cppc_perf.lowest_nonlinear_perf; + lowest_perf = cppc_perf.lowest_perf; + } else { + ret = rdmsrq_safe_on_cpu(cpu, MSR_AMD_CPPC_CAP1, &cap1); + if (ret) { + pr_err("%s read CPPC_CAP1 ret=%d error!\n", __func__, ret); + return ret; + } + + highest_perf = FIELD_GET(AMD_CPPC_HIGHEST_PERF_MASK, cap1); + nominal_perf = FIELD_GET(AMD_CPPC_NOMINAL_PERF_MASK, cap1); + lowest_nonlinear_perf = FIELD_GET(AMD_CPPC_LOWNONLIN_PERF_MASK, cap1); + lowest_perf = FIELD_GET(AMD_CPPC_LOWEST_PERF_MASK, cap1); + } + + cur_perf = READ_ONCE(cpudata->perf); + if (highest_perf != cur_perf.highest_perf && !cpudata->hw_prefcore) { + pr_err("%s cpu%d highest=%d %d highest perf doesn't match\n", + __func__, cpu, highest_perf, cur_perf.highest_perf); + return -EINVAL; + } + if (nominal_perf != cur_perf.nominal_perf || + (lowest_nonlinear_perf != cur_perf.lowest_nonlinear_perf) || + (lowest_perf != cur_perf.lowest_perf)) { + pr_err("%s cpu%d nominal=%d %d lowest_nonlinear=%d %d lowest=%d %d, they should be equal!\n", + __func__, cpu, nominal_perf, cur_perf.nominal_perf, + lowest_nonlinear_perf, cur_perf.lowest_nonlinear_perf, + lowest_perf, cur_perf.lowest_perf); + return -EINVAL; + } + + if (!((highest_perf >= nominal_perf) && + (nominal_perf > lowest_nonlinear_perf) && + (lowest_nonlinear_perf >= lowest_perf) && + (lowest_perf > 0))) { + pr_err("%s cpu%d highest=%d >= nominal=%d > lowest_nonlinear=%d > lowest=%d > 0, the formula is incorrect!\n", + __func__, cpu, highest_perf, nominal_perf, + lowest_nonlinear_perf, lowest_perf); + return -EINVAL; + } + } + + return 0; +} + +/* + * Check if frequency values are reasonable. + * max_freq >= nominal_freq > lowest_nonlinear_freq > min_freq > 0 + * check max freq when set support boost mode. + */ +static int amd_pstate_ut_check_freq(u32 index) +{ + int cpu = 0; + + for_each_online_cpu(cpu) { + struct cpufreq_policy *policy __free(put_cpufreq_policy) = NULL; + struct amd_cpudata *cpudata; + + policy = cpufreq_cpu_get(cpu); + if (!policy) + continue; + cpudata = policy->driver_data; + + if (!((policy->cpuinfo.max_freq >= cpudata->nominal_freq) && + (cpudata->nominal_freq > cpudata->lowest_nonlinear_freq) && + (cpudata->lowest_nonlinear_freq >= policy->cpuinfo.min_freq) && + (policy->cpuinfo.min_freq > 0))) { + pr_err("%s cpu%d max=%d >= nominal=%d > lowest_nonlinear=%d > min=%d > 0, the formula is incorrect!\n", + __func__, cpu, policy->cpuinfo.max_freq, cpudata->nominal_freq, + cpudata->lowest_nonlinear_freq, policy->cpuinfo.min_freq); + return -EINVAL; + } + + if (cpudata->lowest_nonlinear_freq != policy->min) { + pr_err("%s cpu%d cpudata_lowest_nonlinear_freq=%d policy_min=%d, they should be equal!\n", + __func__, cpu, cpudata->lowest_nonlinear_freq, policy->min); + return -EINVAL; + } + + if (cpudata->boost_supported) { + if ((policy->max != policy->cpuinfo.max_freq) && + (policy->max != cpudata->nominal_freq)) { + pr_err("%s cpu%d policy_max=%d should be equal cpu_max=%d or cpu_nominal=%d !\n", + __func__, cpu, policy->max, policy->cpuinfo.max_freq, + cpudata->nominal_freq); + return -EINVAL; + } + } else { + pr_err("%s cpu%d must support boost!\n", __func__, cpu); + return -EINVAL; + } + } + + return 0; +} + +static int amd_pstate_set_mode(enum amd_pstate_mode mode) +{ + const char *mode_str = amd_pstate_get_mode_string(mode); + + pr_debug("->setting mode to %s\n", mode_str); + + return amd_pstate_update_status(mode_str, strlen(mode_str)); +} + +static int amd_pstate_ut_check_driver(u32 index) +{ + enum amd_pstate_mode mode1, mode2 = AMD_PSTATE_DISABLE; + enum amd_pstate_mode orig_mode = amd_pstate_get_status(); + int ret; + + for (mode1 = AMD_PSTATE_DISABLE; mode1 < AMD_PSTATE_MAX; mode1++) { + ret = amd_pstate_set_mode(mode1); + if (ret) + return ret; + for (mode2 = AMD_PSTATE_DISABLE; mode2 < AMD_PSTATE_MAX; mode2++) { + if (mode1 == mode2) + continue; + ret = amd_pstate_set_mode(mode2); + if (ret) + goto out; + } + } + +out: + if (ret) + pr_warn("%s: failed to update status for %s->%s: %d\n", __func__, + amd_pstate_get_mode_string(mode1), + amd_pstate_get_mode_string(mode2), ret); + + amd_pstate_set_mode(orig_mode); + return ret; +} + +static int __init amd_pstate_ut_init(void) +{ + u32 i = 0, arr_size = ARRAY_SIZE(amd_pstate_ut_cases); + + for (i = 0; i < arr_size; i++) { + int ret = amd_pstate_ut_cases[i].func(i); + + if (ret) + pr_err("%-4d %-20s\t fail: %d!\n", i+1, amd_pstate_ut_cases[i].name, ret); + else + pr_info("%-4d %-20s\t success!\n", i+1, amd_pstate_ut_cases[i].name); + } + + return 0; +} + +static void __exit amd_pstate_ut_exit(void) +{ +} + +module_init(amd_pstate_ut_init); +module_exit(amd_pstate_ut_exit); + +MODULE_AUTHOR("Meng Li <li.meng@amd.com>"); +MODULE_DESCRIPTION("AMD P-state driver Test module"); +MODULE_LICENSE("GPL"); diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c new file mode 100644 index 000000000000..c45bc98721d2 --- /dev/null +++ b/drivers/cpufreq/amd-pstate.c @@ -0,0 +1,1880 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * amd-pstate.c - AMD Processor P-state Frequency Driver + * + * Copyright (C) 2021 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Author: Huang Rui <ray.huang@amd.com> + * + * AMD P-State introduces a new CPU performance scaling design for AMD + * processors using the ACPI Collaborative Performance and Power Control (CPPC) + * feature which works with the AMD SMU firmware providing a finer grained + * frequency control range. It is to replace the legacy ACPI P-States control, + * allows a flexible, low-latency interface for the Linux kernel to directly + * communicate the performance hints to hardware. + * + * AMD P-State is supported on recent AMD Zen base CPU series include some of + * Zen2 and Zen3 processors. _CPC needs to be present in the ACPI tables of AMD + * P-State supported system. And there are two types of hardware implementations + * for AMD P-State: 1) Full MSR Solution and 2) Shared Memory Solution. + * X86_FEATURE_CPPC CPU feature flag is used to distinguish the different types. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/bitfield.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/init.h> +#include <linux/smp.h> +#include <linux/sched.h> +#include <linux/cpufreq.h> +#include <linux/compiler.h> +#include <linux/dmi.h> +#include <linux/slab.h> +#include <linux/acpi.h> +#include <linux/io.h> +#include <linux/delay.h> +#include <linux/uaccess.h> +#include <linux/static_call.h> +#include <linux/topology.h> + +#include <acpi/processor.h> +#include <acpi/cppc_acpi.h> + +#include <asm/msr.h> +#include <asm/processor.h> +#include <asm/cpufeature.h> +#include <asm/cpu_device_id.h> + +#include "amd-pstate.h" +#include "amd-pstate-trace.h" + +#define AMD_PSTATE_TRANSITION_LATENCY 20000 +#define AMD_PSTATE_TRANSITION_DELAY 1000 +#define AMD_PSTATE_FAST_CPPC_TRANSITION_DELAY 600 + +#define AMD_CPPC_EPP_PERFORMANCE 0x00 +#define AMD_CPPC_EPP_BALANCE_PERFORMANCE 0x80 +#define AMD_CPPC_EPP_BALANCE_POWERSAVE 0xBF +#define AMD_CPPC_EPP_POWERSAVE 0xFF + +static const char * const amd_pstate_mode_string[] = { + [AMD_PSTATE_UNDEFINED] = "undefined", + [AMD_PSTATE_DISABLE] = "disable", + [AMD_PSTATE_PASSIVE] = "passive", + [AMD_PSTATE_ACTIVE] = "active", + [AMD_PSTATE_GUIDED] = "guided", +}; +static_assert(ARRAY_SIZE(amd_pstate_mode_string) == AMD_PSTATE_MAX); + +const char *amd_pstate_get_mode_string(enum amd_pstate_mode mode) +{ + if (mode < AMD_PSTATE_UNDEFINED || mode >= AMD_PSTATE_MAX) + mode = AMD_PSTATE_UNDEFINED; + return amd_pstate_mode_string[mode]; +} +EXPORT_SYMBOL_GPL(amd_pstate_get_mode_string); + +struct quirk_entry { + u32 nominal_freq; + u32 lowest_freq; +}; + +static struct cpufreq_driver *current_pstate_driver; +static struct cpufreq_driver amd_pstate_driver; +static struct cpufreq_driver amd_pstate_epp_driver; +static int cppc_state = AMD_PSTATE_UNDEFINED; +static bool amd_pstate_prefcore = true; +static struct quirk_entry *quirks; + +/* + * AMD Energy Preference Performance (EPP) + * The EPP is used in the CCLK DPM controller to drive + * the frequency that a core is going to operate during + * short periods of activity. EPP values will be utilized for + * different OS profiles (balanced, performance, power savings) + * display strings corresponding to EPP index in the + * energy_perf_strings[] + * index String + *------------------------------------- + * 0 default + * 1 performance + * 2 balance_performance + * 3 balance_power + * 4 power + */ +enum energy_perf_value_index { + EPP_INDEX_DEFAULT = 0, + EPP_INDEX_PERFORMANCE, + EPP_INDEX_BALANCE_PERFORMANCE, + EPP_INDEX_BALANCE_POWERSAVE, + EPP_INDEX_POWERSAVE, + EPP_INDEX_MAX, +}; + +static const char * const energy_perf_strings[] = { + [EPP_INDEX_DEFAULT] = "default", + [EPP_INDEX_PERFORMANCE] = "performance", + [EPP_INDEX_BALANCE_PERFORMANCE] = "balance_performance", + [EPP_INDEX_BALANCE_POWERSAVE] = "balance_power", + [EPP_INDEX_POWERSAVE] = "power", +}; +static_assert(ARRAY_SIZE(energy_perf_strings) == EPP_INDEX_MAX); + +static unsigned int epp_values[] = { + [EPP_INDEX_DEFAULT] = 0, + [EPP_INDEX_PERFORMANCE] = AMD_CPPC_EPP_PERFORMANCE, + [EPP_INDEX_BALANCE_PERFORMANCE] = AMD_CPPC_EPP_BALANCE_PERFORMANCE, + [EPP_INDEX_BALANCE_POWERSAVE] = AMD_CPPC_EPP_BALANCE_POWERSAVE, + [EPP_INDEX_POWERSAVE] = AMD_CPPC_EPP_POWERSAVE, +}; +static_assert(ARRAY_SIZE(epp_values) == EPP_INDEX_MAX); + +typedef int (*cppc_mode_transition_fn)(int); + +static struct quirk_entry quirk_amd_7k62 = { + .nominal_freq = 2600, + .lowest_freq = 550, +}; + +static inline u8 freq_to_perf(union perf_cached perf, u32 nominal_freq, unsigned int freq_val) +{ + u32 perf_val = DIV_ROUND_UP_ULL((u64)freq_val * perf.nominal_perf, nominal_freq); + + return (u8)clamp(perf_val, perf.lowest_perf, perf.highest_perf); +} + +static inline u32 perf_to_freq(union perf_cached perf, u32 nominal_freq, u8 perf_val) +{ + return DIV_ROUND_UP_ULL((u64)nominal_freq * perf_val, + perf.nominal_perf); +} + +static int __init dmi_matched_7k62_bios_bug(const struct dmi_system_id *dmi) +{ + /** + * match the broken bios for family 17h processor support CPPC V2 + * broken BIOS lack of nominal_freq and lowest_freq capabilities + * definition in ACPI tables + */ + if (cpu_feature_enabled(X86_FEATURE_ZEN2)) { + quirks = dmi->driver_data; + pr_info("Overriding nominal and lowest frequencies for %s\n", dmi->ident); + return 1; + } + + return 0; +} + +static const struct dmi_system_id amd_pstate_quirks_table[] __initconst = { + { + .callback = dmi_matched_7k62_bios_bug, + .ident = "AMD EPYC 7K62", + .matches = { + DMI_MATCH(DMI_BIOS_VERSION, "5.14"), + DMI_MATCH(DMI_BIOS_RELEASE, "12/12/2019"), + }, + .driver_data = &quirk_amd_7k62, + }, + {} +}; +MODULE_DEVICE_TABLE(dmi, amd_pstate_quirks_table); + +static inline int get_mode_idx_from_str(const char *str, size_t size) +{ + int i; + + for (i = 0; i < AMD_PSTATE_MAX; i++) { + if (!strncmp(str, amd_pstate_mode_string[i], size)) + return i; + } + return -EINVAL; +} + +static DEFINE_MUTEX(amd_pstate_driver_lock); + +static u8 msr_get_epp(struct amd_cpudata *cpudata) +{ + u64 value; + int ret; + + ret = rdmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value); + if (ret < 0) { + pr_debug("Could not retrieve energy perf value (%d)\n", ret); + return ret; + } + + return FIELD_GET(AMD_CPPC_EPP_PERF_MASK, value); +} + +DEFINE_STATIC_CALL(amd_pstate_get_epp, msr_get_epp); + +static inline s16 amd_pstate_get_epp(struct amd_cpudata *cpudata) +{ + return static_call(amd_pstate_get_epp)(cpudata); +} + +static u8 shmem_get_epp(struct amd_cpudata *cpudata) +{ + u64 epp; + int ret; + + ret = cppc_get_epp_perf(cpudata->cpu, &epp); + if (ret < 0) { + pr_debug("Could not retrieve energy perf value (%d)\n", ret); + return ret; + } + + return FIELD_GET(AMD_CPPC_EPP_PERF_MASK, epp); +} + +static int msr_update_perf(struct cpufreq_policy *policy, u8 min_perf, + u8 des_perf, u8 max_perf, u8 epp, bool fast_switch) +{ + struct amd_cpudata *cpudata = policy->driver_data; + u64 value, prev; + + value = prev = READ_ONCE(cpudata->cppc_req_cached); + + value &= ~(AMD_CPPC_MAX_PERF_MASK | AMD_CPPC_MIN_PERF_MASK | + AMD_CPPC_DES_PERF_MASK | AMD_CPPC_EPP_PERF_MASK); + value |= FIELD_PREP(AMD_CPPC_MAX_PERF_MASK, max_perf); + value |= FIELD_PREP(AMD_CPPC_DES_PERF_MASK, des_perf); + value |= FIELD_PREP(AMD_CPPC_MIN_PERF_MASK, min_perf); + value |= FIELD_PREP(AMD_CPPC_EPP_PERF_MASK, epp); + + if (trace_amd_pstate_epp_perf_enabled()) { + union perf_cached perf = READ_ONCE(cpudata->perf); + + trace_amd_pstate_epp_perf(cpudata->cpu, + perf.highest_perf, + epp, + min_perf, + max_perf, + policy->boost_enabled, + value != prev); + } + + if (value == prev) + return 0; + + if (fast_switch) { + wrmsrq(MSR_AMD_CPPC_REQ, value); + return 0; + } else { + int ret = wrmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); + + if (ret) + return ret; + } + + WRITE_ONCE(cpudata->cppc_req_cached, value); + + return 0; +} + +DEFINE_STATIC_CALL(amd_pstate_update_perf, msr_update_perf); + +static inline int amd_pstate_update_perf(struct cpufreq_policy *policy, + u8 min_perf, u8 des_perf, + u8 max_perf, u8 epp, + bool fast_switch) +{ + return static_call(amd_pstate_update_perf)(policy, min_perf, des_perf, + max_perf, epp, fast_switch); +} + +static int msr_set_epp(struct cpufreq_policy *policy, u8 epp) +{ + struct amd_cpudata *cpudata = policy->driver_data; + u64 value, prev; + int ret; + + value = prev = READ_ONCE(cpudata->cppc_req_cached); + value &= ~AMD_CPPC_EPP_PERF_MASK; + value |= FIELD_PREP(AMD_CPPC_EPP_PERF_MASK, epp); + + if (trace_amd_pstate_epp_perf_enabled()) { + union perf_cached perf = cpudata->perf; + + trace_amd_pstate_epp_perf(cpudata->cpu, perf.highest_perf, + epp, + FIELD_GET(AMD_CPPC_MIN_PERF_MASK, + cpudata->cppc_req_cached), + FIELD_GET(AMD_CPPC_MAX_PERF_MASK, + cpudata->cppc_req_cached), + policy->boost_enabled, + value != prev); + } + + if (value == prev) + return 0; + + ret = wrmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); + if (ret) { + pr_err("failed to set energy perf value (%d)\n", ret); + return ret; + } + + /* update both so that msr_update_perf() can effectively check */ + WRITE_ONCE(cpudata->cppc_req_cached, value); + + return ret; +} + +DEFINE_STATIC_CALL(amd_pstate_set_epp, msr_set_epp); + +static inline int amd_pstate_set_epp(struct cpufreq_policy *policy, u8 epp) +{ + return static_call(amd_pstate_set_epp)(policy, epp); +} + +static int shmem_set_epp(struct cpufreq_policy *policy, u8 epp) +{ + struct amd_cpudata *cpudata = policy->driver_data; + struct cppc_perf_ctrls perf_ctrls; + u8 epp_cached; + u64 value; + int ret; + + + epp_cached = FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached); + if (trace_amd_pstate_epp_perf_enabled()) { + union perf_cached perf = cpudata->perf; + + trace_amd_pstate_epp_perf(cpudata->cpu, perf.highest_perf, + epp, + FIELD_GET(AMD_CPPC_MIN_PERF_MASK, + cpudata->cppc_req_cached), + FIELD_GET(AMD_CPPC_MAX_PERF_MASK, + cpudata->cppc_req_cached), + policy->boost_enabled, + epp != epp_cached); + } + + if (epp == epp_cached) + return 0; + + perf_ctrls.energy_perf = epp; + ret = cppc_set_epp_perf(cpudata->cpu, &perf_ctrls, 1); + if (ret) { + pr_debug("failed to set energy perf value (%d)\n", ret); + return ret; + } + + value = READ_ONCE(cpudata->cppc_req_cached); + value &= ~AMD_CPPC_EPP_PERF_MASK; + value |= FIELD_PREP(AMD_CPPC_EPP_PERF_MASK, epp); + WRITE_ONCE(cpudata->cppc_req_cached, value); + + return ret; +} + +static inline int msr_cppc_enable(struct cpufreq_policy *policy) +{ + return wrmsrq_safe_on_cpu(policy->cpu, MSR_AMD_CPPC_ENABLE, 1); +} + +static int shmem_cppc_enable(struct cpufreq_policy *policy) +{ + return cppc_set_enable(policy->cpu, 1); +} + +DEFINE_STATIC_CALL(amd_pstate_cppc_enable, msr_cppc_enable); + +static inline int amd_pstate_cppc_enable(struct cpufreq_policy *policy) +{ + return static_call(amd_pstate_cppc_enable)(policy); +} + +static int msr_init_perf(struct amd_cpudata *cpudata) +{ + union perf_cached perf = READ_ONCE(cpudata->perf); + u64 cap1, numerator, cppc_req; + u8 min_perf; + + int ret = rdmsrq_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, + &cap1); + if (ret) + return ret; + + ret = amd_get_boost_ratio_numerator(cpudata->cpu, &numerator); + if (ret) + return ret; + + ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &cppc_req); + if (ret) + return ret; + + WRITE_ONCE(cpudata->cppc_req_cached, cppc_req); + min_perf = FIELD_GET(AMD_CPPC_MIN_PERF_MASK, cppc_req); + + /* + * Clear out the min_perf part to check if the rest of the MSR is 0, if yes, this is an + * indication that the min_perf value is the one specified through the BIOS option + */ + cppc_req &= ~(AMD_CPPC_MIN_PERF_MASK); + + if (!cppc_req) + perf.bios_min_perf = min_perf; + + perf.highest_perf = numerator; + perf.max_limit_perf = numerator; + perf.min_limit_perf = FIELD_GET(AMD_CPPC_LOWEST_PERF_MASK, cap1); + perf.nominal_perf = FIELD_GET(AMD_CPPC_NOMINAL_PERF_MASK, cap1); + perf.lowest_nonlinear_perf = FIELD_GET(AMD_CPPC_LOWNONLIN_PERF_MASK, cap1); + perf.lowest_perf = FIELD_GET(AMD_CPPC_LOWEST_PERF_MASK, cap1); + WRITE_ONCE(cpudata->perf, perf); + WRITE_ONCE(cpudata->prefcore_ranking, FIELD_GET(AMD_CPPC_HIGHEST_PERF_MASK, cap1)); + + return 0; +} + +static int shmem_init_perf(struct amd_cpudata *cpudata) +{ + struct cppc_perf_caps cppc_perf; + union perf_cached perf = READ_ONCE(cpudata->perf); + u64 numerator; + bool auto_sel; + + int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); + if (ret) + return ret; + + ret = amd_get_boost_ratio_numerator(cpudata->cpu, &numerator); + if (ret) + return ret; + + perf.highest_perf = numerator; + perf.max_limit_perf = numerator; + perf.min_limit_perf = cppc_perf.lowest_perf; + perf.nominal_perf = cppc_perf.nominal_perf; + perf.lowest_nonlinear_perf = cppc_perf.lowest_nonlinear_perf; + perf.lowest_perf = cppc_perf.lowest_perf; + WRITE_ONCE(cpudata->perf, perf); + WRITE_ONCE(cpudata->prefcore_ranking, cppc_perf.highest_perf); + + if (cppc_state == AMD_PSTATE_ACTIVE) + return 0; + + ret = cppc_get_auto_sel(cpudata->cpu, &auto_sel); + if (ret) { + pr_warn("failed to get auto_sel, ret: %d\n", ret); + return 0; + } + + ret = cppc_set_auto_sel(cpudata->cpu, + (cppc_state == AMD_PSTATE_PASSIVE) ? 0 : 1); + + if (ret) + pr_warn("failed to set auto_sel, ret: %d\n", ret); + + return ret; +} + +DEFINE_STATIC_CALL(amd_pstate_init_perf, msr_init_perf); + +static inline int amd_pstate_init_perf(struct amd_cpudata *cpudata) +{ + return static_call(amd_pstate_init_perf)(cpudata); +} + +static int shmem_update_perf(struct cpufreq_policy *policy, u8 min_perf, + u8 des_perf, u8 max_perf, u8 epp, bool fast_switch) +{ + struct amd_cpudata *cpudata = policy->driver_data; + struct cppc_perf_ctrls perf_ctrls; + u64 value, prev; + int ret; + + if (cppc_state == AMD_PSTATE_ACTIVE) { + int ret = shmem_set_epp(policy, epp); + + if (ret) + return ret; + } + + value = prev = READ_ONCE(cpudata->cppc_req_cached); + + value &= ~(AMD_CPPC_MAX_PERF_MASK | AMD_CPPC_MIN_PERF_MASK | + AMD_CPPC_DES_PERF_MASK | AMD_CPPC_EPP_PERF_MASK); + value |= FIELD_PREP(AMD_CPPC_MAX_PERF_MASK, max_perf); + value |= FIELD_PREP(AMD_CPPC_DES_PERF_MASK, des_perf); + value |= FIELD_PREP(AMD_CPPC_MIN_PERF_MASK, min_perf); + value |= FIELD_PREP(AMD_CPPC_EPP_PERF_MASK, epp); + + if (trace_amd_pstate_epp_perf_enabled()) { + union perf_cached perf = READ_ONCE(cpudata->perf); + + trace_amd_pstate_epp_perf(cpudata->cpu, + perf.highest_perf, + epp, + min_perf, + max_perf, + policy->boost_enabled, + value != prev); + } + + if (value == prev) + return 0; + + perf_ctrls.max_perf = max_perf; + perf_ctrls.min_perf = min_perf; + perf_ctrls.desired_perf = des_perf; + + ret = cppc_set_perf(cpudata->cpu, &perf_ctrls); + if (ret) + return ret; + + WRITE_ONCE(cpudata->cppc_req_cached, value); + + return 0; +} + +static inline bool amd_pstate_sample(struct amd_cpudata *cpudata) +{ + u64 aperf, mperf, tsc; + unsigned long flags; + + local_irq_save(flags); + rdmsrq(MSR_IA32_APERF, aperf); + rdmsrq(MSR_IA32_MPERF, mperf); + tsc = rdtsc(); + + if (cpudata->prev.mperf == mperf || cpudata->prev.tsc == tsc) { + local_irq_restore(flags); + return false; + } + + local_irq_restore(flags); + + cpudata->cur.aperf = aperf; + cpudata->cur.mperf = mperf; + cpudata->cur.tsc = tsc; + cpudata->cur.aperf -= cpudata->prev.aperf; + cpudata->cur.mperf -= cpudata->prev.mperf; + cpudata->cur.tsc -= cpudata->prev.tsc; + + cpudata->prev.aperf = aperf; + cpudata->prev.mperf = mperf; + cpudata->prev.tsc = tsc; + + cpudata->freq = div64_u64((cpudata->cur.aperf * cpu_khz), cpudata->cur.mperf); + + return true; +} + +static void amd_pstate_update(struct amd_cpudata *cpudata, u8 min_perf, + u8 des_perf, u8 max_perf, bool fast_switch, int gov_flags) +{ + struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpudata->cpu); + union perf_cached perf = READ_ONCE(cpudata->perf); + + if (!policy) + return; + + /* limit the max perf when core performance boost feature is disabled */ + if (!cpudata->boost_supported) + max_perf = min_t(u8, perf.nominal_perf, max_perf); + + des_perf = clamp_t(u8, des_perf, min_perf, max_perf); + + policy->cur = perf_to_freq(perf, cpudata->nominal_freq, des_perf); + + if ((cppc_state == AMD_PSTATE_GUIDED) && (gov_flags & CPUFREQ_GOV_DYNAMIC_SWITCHING)) { + min_perf = des_perf; + des_perf = 0; + } + + if (trace_amd_pstate_perf_enabled() && amd_pstate_sample(cpudata)) { + trace_amd_pstate_perf(min_perf, des_perf, max_perf, cpudata->freq, + cpudata->cur.mperf, cpudata->cur.aperf, cpudata->cur.tsc, + cpudata->cpu, fast_switch); + } + + amd_pstate_update_perf(policy, min_perf, des_perf, max_perf, 0, fast_switch); +} + +static int amd_pstate_verify(struct cpufreq_policy_data *policy_data) +{ + /* + * Initialize lower frequency limit (i.e.policy->min) with + * lowest_nonlinear_frequency or the min frequency (if) specified in BIOS, + * Override the initial value set by cpufreq core and amd-pstate qos_requests. + */ + if (policy_data->min == FREQ_QOS_MIN_DEFAULT_VALUE) { + struct cpufreq_policy *policy __free(put_cpufreq_policy) = + cpufreq_cpu_get(policy_data->cpu); + struct amd_cpudata *cpudata; + union perf_cached perf; + + if (!policy) + return -EINVAL; + + cpudata = policy->driver_data; + perf = READ_ONCE(cpudata->perf); + + if (perf.bios_min_perf) + policy_data->min = perf_to_freq(perf, cpudata->nominal_freq, + perf.bios_min_perf); + else + policy_data->min = cpudata->lowest_nonlinear_freq; + } + + cpufreq_verify_within_cpu_limits(policy_data); + + return 0; +} + +static void amd_pstate_update_min_max_limit(struct cpufreq_policy *policy) +{ + struct amd_cpudata *cpudata = policy->driver_data; + union perf_cached perf = READ_ONCE(cpudata->perf); + + perf.max_limit_perf = freq_to_perf(perf, cpudata->nominal_freq, policy->max); + WRITE_ONCE(cpudata->max_limit_freq, policy->max); + + if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { + perf.min_limit_perf = min(perf.nominal_perf, perf.max_limit_perf); + WRITE_ONCE(cpudata->min_limit_freq, min(cpudata->nominal_freq, cpudata->max_limit_freq)); + } else { + perf.min_limit_perf = freq_to_perf(perf, cpudata->nominal_freq, policy->min); + WRITE_ONCE(cpudata->min_limit_freq, policy->min); + } + + WRITE_ONCE(cpudata->perf, perf); +} + +static int amd_pstate_update_freq(struct cpufreq_policy *policy, + unsigned int target_freq, bool fast_switch) +{ + struct cpufreq_freqs freqs; + struct amd_cpudata *cpudata; + union perf_cached perf; + u8 des_perf; + + cpudata = policy->driver_data; + + if (policy->min != cpudata->min_limit_freq || policy->max != cpudata->max_limit_freq) + amd_pstate_update_min_max_limit(policy); + + perf = READ_ONCE(cpudata->perf); + + freqs.old = policy->cur; + freqs.new = target_freq; + + des_perf = freq_to_perf(perf, cpudata->nominal_freq, target_freq); + + WARN_ON(fast_switch && !policy->fast_switch_enabled); + /* + * If fast_switch is desired, then there aren't any registered + * transition notifiers. See comment for + * cpufreq_enable_fast_switch(). + */ + if (!fast_switch) + cpufreq_freq_transition_begin(policy, &freqs); + + amd_pstate_update(cpudata, perf.min_limit_perf, des_perf, + perf.max_limit_perf, fast_switch, + policy->governor->flags); + + if (!fast_switch) + cpufreq_freq_transition_end(policy, &freqs, false); + + return 0; +} + +static int amd_pstate_target(struct cpufreq_policy *policy, + unsigned int target_freq, + unsigned int relation) +{ + return amd_pstate_update_freq(policy, target_freq, false); +} + +static unsigned int amd_pstate_fast_switch(struct cpufreq_policy *policy, + unsigned int target_freq) +{ + if (!amd_pstate_update_freq(policy, target_freq, true)) + return target_freq; + return policy->cur; +} + +static void amd_pstate_adjust_perf(unsigned int cpu, + unsigned long _min_perf, + unsigned long target_perf, + unsigned long capacity) +{ + u8 max_perf, min_perf, des_perf, cap_perf; + struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu); + struct amd_cpudata *cpudata; + union perf_cached perf; + + if (!policy) + return; + + cpudata = policy->driver_data; + + if (policy->min != cpudata->min_limit_freq || policy->max != cpudata->max_limit_freq) + amd_pstate_update_min_max_limit(policy); + + perf = READ_ONCE(cpudata->perf); + cap_perf = perf.highest_perf; + + des_perf = cap_perf; + if (target_perf < capacity) + des_perf = DIV_ROUND_UP(cap_perf * target_perf, capacity); + + if (_min_perf < capacity) + min_perf = DIV_ROUND_UP(cap_perf * _min_perf, capacity); + else + min_perf = cap_perf; + + if (min_perf < perf.min_limit_perf) + min_perf = perf.min_limit_perf; + + max_perf = perf.max_limit_perf; + if (max_perf < min_perf) + max_perf = min_perf; + + amd_pstate_update(cpudata, min_perf, des_perf, max_perf, true, + policy->governor->flags); +} + +static int amd_pstate_cpu_boost_update(struct cpufreq_policy *policy, bool on) +{ + struct amd_cpudata *cpudata = policy->driver_data; + union perf_cached perf = READ_ONCE(cpudata->perf); + u32 nominal_freq, max_freq; + int ret = 0; + + nominal_freq = READ_ONCE(cpudata->nominal_freq); + max_freq = perf_to_freq(perf, cpudata->nominal_freq, perf.highest_perf); + + if (on) + policy->cpuinfo.max_freq = max_freq; + else if (policy->cpuinfo.max_freq > nominal_freq) + policy->cpuinfo.max_freq = nominal_freq; + + policy->max = policy->cpuinfo.max_freq; + + if (cppc_state == AMD_PSTATE_PASSIVE) { + ret = freq_qos_update_request(&cpudata->req[1], policy->cpuinfo.max_freq); + if (ret < 0) + pr_debug("Failed to update freq constraint: CPU%d\n", cpudata->cpu); + } + + return ret < 0 ? ret : 0; +} + +static int amd_pstate_set_boost(struct cpufreq_policy *policy, int state) +{ + struct amd_cpudata *cpudata = policy->driver_data; + int ret; + + if (!cpudata->boost_supported) { + pr_err("Boost mode is not supported by this processor or SBIOS\n"); + return -EOPNOTSUPP; + } + + ret = amd_pstate_cpu_boost_update(policy, state); + refresh_frequency_limits(policy); + + return ret; +} + +static int amd_pstate_init_boost_support(struct amd_cpudata *cpudata) +{ + u64 boost_val; + int ret = -1; + + /* + * If platform has no CPB support or disable it, initialize current driver + * boost_enabled state to be false, it is not an error for cpufreq core to handle. + */ + if (!cpu_feature_enabled(X86_FEATURE_CPB)) { + pr_debug_once("Boost CPB capabilities not present in the processor\n"); + ret = 0; + goto exit_err; + } + + ret = rdmsrq_on_cpu(cpudata->cpu, MSR_K7_HWCR, &boost_val); + if (ret) { + pr_err_once("failed to read initial CPU boost state!\n"); + ret = -EIO; + goto exit_err; + } + + if (!(boost_val & MSR_K7_HWCR_CPB_DIS)) + cpudata->boost_supported = true; + + return 0; + +exit_err: + cpudata->boost_supported = false; + return ret; +} + +static void amd_perf_ctl_reset(unsigned int cpu) +{ + wrmsrq_on_cpu(cpu, MSR_AMD_PERF_CTL, 0); +} + +#define CPPC_MAX_PERF U8_MAX + +static void amd_pstate_init_prefcore(struct amd_cpudata *cpudata) +{ + /* user disabled or not detected */ + if (!amd_pstate_prefcore) + return; + + /* should use amd-hfi instead */ + if (cpu_feature_enabled(X86_FEATURE_AMD_WORKLOAD_CLASS) && + IS_ENABLED(CONFIG_AMD_HFI)) { + amd_pstate_prefcore = false; + return; + } + + cpudata->hw_prefcore = true; + + /* Priorities must be initialized before ITMT support can be toggled on. */ + sched_set_itmt_core_prio((int)READ_ONCE(cpudata->prefcore_ranking), cpudata->cpu); +} + +static void amd_pstate_update_limits(struct cpufreq_policy *policy) +{ + struct amd_cpudata *cpudata; + u32 prev_high = 0, cur_high = 0; + bool highest_perf_changed = false; + unsigned int cpu = policy->cpu; + + if (!amd_pstate_prefcore) + return; + + if (amd_get_highest_perf(cpu, &cur_high)) + return; + + cpudata = policy->driver_data; + + prev_high = READ_ONCE(cpudata->prefcore_ranking); + highest_perf_changed = (prev_high != cur_high); + if (highest_perf_changed) { + WRITE_ONCE(cpudata->prefcore_ranking, cur_high); + + if (cur_high < CPPC_MAX_PERF) { + sched_set_itmt_core_prio((int)cur_high, cpu); + sched_update_asym_prefer_cpu(cpu, prev_high, cur_high); + } + } +} + +/* + * Get pstate transition delay time from ACPI tables that firmware set + * instead of using hardcode value directly. + */ +static u32 amd_pstate_get_transition_delay_us(unsigned int cpu) +{ + int transition_delay_ns; + + transition_delay_ns = cppc_get_transition_latency(cpu); + if (transition_delay_ns < 0) { + if (cpu_feature_enabled(X86_FEATURE_AMD_FAST_CPPC)) + return AMD_PSTATE_FAST_CPPC_TRANSITION_DELAY; + else + return AMD_PSTATE_TRANSITION_DELAY; + } + + return transition_delay_ns / NSEC_PER_USEC; +} + +/* + * Get pstate transition latency value from ACPI tables that firmware + * set instead of using hardcode value directly. + */ +static u32 amd_pstate_get_transition_latency(unsigned int cpu) +{ + int transition_latency; + + transition_latency = cppc_get_transition_latency(cpu); + if (transition_latency < 0) + return AMD_PSTATE_TRANSITION_LATENCY; + + return transition_latency; +} + +/* + * amd_pstate_init_freq: Initialize the nominal_freq and lowest_nonlinear_freq + * for the @cpudata object. + * + * Requires: all perf members of @cpudata to be initialized. + * + * Returns 0 on success, non-zero value on failure. + */ +static int amd_pstate_init_freq(struct amd_cpudata *cpudata) +{ + u32 min_freq, max_freq, nominal_freq, lowest_nonlinear_freq; + struct cppc_perf_caps cppc_perf; + union perf_cached perf; + int ret; + + ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); + if (ret) + return ret; + perf = READ_ONCE(cpudata->perf); + + if (quirks && quirks->nominal_freq) + nominal_freq = quirks->nominal_freq; + else + nominal_freq = cppc_perf.nominal_freq; + nominal_freq *= 1000; + + if (quirks && quirks->lowest_freq) { + min_freq = quirks->lowest_freq; + perf.lowest_perf = freq_to_perf(perf, nominal_freq, min_freq); + WRITE_ONCE(cpudata->perf, perf); + } else + min_freq = cppc_perf.lowest_freq; + + min_freq *= 1000; + + WRITE_ONCE(cpudata->nominal_freq, nominal_freq); + + max_freq = perf_to_freq(perf, nominal_freq, perf.highest_perf); + lowest_nonlinear_freq = perf_to_freq(perf, nominal_freq, perf.lowest_nonlinear_perf); + WRITE_ONCE(cpudata->lowest_nonlinear_freq, lowest_nonlinear_freq); + + /** + * Below values need to be initialized correctly, otherwise driver will fail to load + * max_freq is calculated according to (nominal_freq * highest_perf)/nominal_perf + * lowest_nonlinear_freq is a value between [min_freq, nominal_freq] + * Check _CPC in ACPI table objects if any values are incorrect + */ + if (min_freq <= 0 || max_freq <= 0 || nominal_freq <= 0 || min_freq > max_freq) { + pr_err("min_freq(%d) or max_freq(%d) or nominal_freq(%d) value is incorrect\n", + min_freq, max_freq, nominal_freq); + return -EINVAL; + } + + if (lowest_nonlinear_freq <= min_freq || lowest_nonlinear_freq > nominal_freq) { + pr_err("lowest_nonlinear_freq(%d) value is out of range [min_freq(%d), nominal_freq(%d)]\n", + lowest_nonlinear_freq, min_freq, nominal_freq); + return -EINVAL; + } + + return 0; +} + +static int amd_pstate_cpu_init(struct cpufreq_policy *policy) +{ + struct amd_cpudata *cpudata; + union perf_cached perf; + struct device *dev; + int ret; + + /* + * Resetting PERF_CTL_MSR will put the CPU in P0 frequency, + * which is ideal for initialization process. + */ + amd_perf_ctl_reset(policy->cpu); + dev = get_cpu_device(policy->cpu); + if (!dev) + return -ENODEV; + + cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL); + if (!cpudata) + return -ENOMEM; + + cpudata->cpu = policy->cpu; + + ret = amd_pstate_init_perf(cpudata); + if (ret) + goto free_cpudata1; + + amd_pstate_init_prefcore(cpudata); + + ret = amd_pstate_init_freq(cpudata); + if (ret) + goto free_cpudata1; + + ret = amd_pstate_init_boost_support(cpudata); + if (ret) + goto free_cpudata1; + + policy->cpuinfo.transition_latency = amd_pstate_get_transition_latency(policy->cpu); + policy->transition_delay_us = amd_pstate_get_transition_delay_us(policy->cpu); + + perf = READ_ONCE(cpudata->perf); + + policy->cpuinfo.min_freq = policy->min = perf_to_freq(perf, + cpudata->nominal_freq, + perf.lowest_perf); + policy->cpuinfo.max_freq = policy->max = perf_to_freq(perf, + cpudata->nominal_freq, + perf.highest_perf); + + ret = amd_pstate_cppc_enable(policy); + if (ret) + goto free_cpudata1; + + policy->boost_supported = READ_ONCE(cpudata->boost_supported); + + /* It will be updated by governor */ + policy->cur = policy->cpuinfo.min_freq; + + if (cpu_feature_enabled(X86_FEATURE_CPPC)) + policy->fast_switch_possible = true; + + ret = freq_qos_add_request(&policy->constraints, &cpudata->req[0], + FREQ_QOS_MIN, FREQ_QOS_MIN_DEFAULT_VALUE); + if (ret < 0) { + dev_err(dev, "Failed to add min-freq constraint (%d)\n", ret); + goto free_cpudata1; + } + + ret = freq_qos_add_request(&policy->constraints, &cpudata->req[1], + FREQ_QOS_MAX, policy->cpuinfo.max_freq); + if (ret < 0) { + dev_err(dev, "Failed to add max-freq constraint (%d)\n", ret); + goto free_cpudata2; + } + + policy->driver_data = cpudata; + + if (!current_pstate_driver->adjust_perf) + current_pstate_driver->adjust_perf = amd_pstate_adjust_perf; + + return 0; + +free_cpudata2: + freq_qos_remove_request(&cpudata->req[0]); +free_cpudata1: + pr_warn("Failed to initialize CPU %d: %d\n", policy->cpu, ret); + kfree(cpudata); + return ret; +} + +static void amd_pstate_cpu_exit(struct cpufreq_policy *policy) +{ + struct amd_cpudata *cpudata = policy->driver_data; + union perf_cached perf = READ_ONCE(cpudata->perf); + + /* Reset CPPC_REQ MSR to the BIOS value */ + amd_pstate_update_perf(policy, perf.bios_min_perf, 0U, 0U, 0U, false); + + freq_qos_remove_request(&cpudata->req[1]); + freq_qos_remove_request(&cpudata->req[0]); + policy->fast_switch_possible = false; + kfree(cpudata); +} + +/* Sysfs attributes */ + +/* + * This frequency is to indicate the maximum hardware frequency. + * If boost is not active but supported, the frequency will be larger than the + * one in cpuinfo. + */ +static ssize_t show_amd_pstate_max_freq(struct cpufreq_policy *policy, + char *buf) +{ + struct amd_cpudata *cpudata; + union perf_cached perf; + + cpudata = policy->driver_data; + perf = READ_ONCE(cpudata->perf); + + return sysfs_emit(buf, "%u\n", + perf_to_freq(perf, cpudata->nominal_freq, perf.highest_perf)); +} + +static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *policy, + char *buf) +{ + struct amd_cpudata *cpudata; + union perf_cached perf; + + cpudata = policy->driver_data; + perf = READ_ONCE(cpudata->perf); + + return sysfs_emit(buf, "%u\n", + perf_to_freq(perf, cpudata->nominal_freq, perf.lowest_nonlinear_perf)); +} + +/* + * In some of ASICs, the highest_perf is not the one in the _CPC table, so we + * need to expose it to sysfs. + */ +static ssize_t show_amd_pstate_highest_perf(struct cpufreq_policy *policy, + char *buf) +{ + struct amd_cpudata *cpudata; + + cpudata = policy->driver_data; + + return sysfs_emit(buf, "%u\n", cpudata->perf.highest_perf); +} + +static ssize_t show_amd_pstate_prefcore_ranking(struct cpufreq_policy *policy, + char *buf) +{ + u8 perf; + struct amd_cpudata *cpudata = policy->driver_data; + + perf = READ_ONCE(cpudata->prefcore_ranking); + + return sysfs_emit(buf, "%u\n", perf); +} + +static ssize_t show_amd_pstate_hw_prefcore(struct cpufreq_policy *policy, + char *buf) +{ + bool hw_prefcore; + struct amd_cpudata *cpudata = policy->driver_data; + + hw_prefcore = READ_ONCE(cpudata->hw_prefcore); + + return sysfs_emit(buf, "%s\n", str_enabled_disabled(hw_prefcore)); +} + +static ssize_t show_energy_performance_available_preferences( + struct cpufreq_policy *policy, char *buf) +{ + int offset = 0, i; + struct amd_cpudata *cpudata = policy->driver_data; + + if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) + return sysfs_emit_at(buf, offset, "%s\n", + energy_perf_strings[EPP_INDEX_PERFORMANCE]); + + for (i = 0; i < ARRAY_SIZE(energy_perf_strings); i++) + offset += sysfs_emit_at(buf, offset, "%s ", energy_perf_strings[i]); + + offset += sysfs_emit_at(buf, offset, "\n"); + + return offset; +} + +static ssize_t store_energy_performance_preference( + struct cpufreq_policy *policy, const char *buf, size_t count) +{ + struct amd_cpudata *cpudata = policy->driver_data; + ssize_t ret; + u8 epp; + + ret = sysfs_match_string(energy_perf_strings, buf); + if (ret < 0) + return -EINVAL; + + if (!ret) + epp = cpudata->epp_default; + else + epp = epp_values[ret]; + + if (epp > 0 && policy->policy == CPUFREQ_POLICY_PERFORMANCE) { + pr_debug("EPP cannot be set under performance policy\n"); + return -EBUSY; + } + + ret = amd_pstate_set_epp(policy, epp); + + return ret ? ret : count; +} + +static ssize_t show_energy_performance_preference( + struct cpufreq_policy *policy, char *buf) +{ + struct amd_cpudata *cpudata = policy->driver_data; + u8 preference, epp; + + epp = FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached); + + switch (epp) { + case AMD_CPPC_EPP_PERFORMANCE: + preference = EPP_INDEX_PERFORMANCE; + break; + case AMD_CPPC_EPP_BALANCE_PERFORMANCE: + preference = EPP_INDEX_BALANCE_PERFORMANCE; + break; + case AMD_CPPC_EPP_BALANCE_POWERSAVE: + preference = EPP_INDEX_BALANCE_POWERSAVE; + break; + case AMD_CPPC_EPP_POWERSAVE: + preference = EPP_INDEX_POWERSAVE; + break; + default: + return -EINVAL; + } + + return sysfs_emit(buf, "%s\n", energy_perf_strings[preference]); +} + +static void amd_pstate_driver_cleanup(void) +{ + if (amd_pstate_prefcore) + sched_clear_itmt_support(); + + cppc_state = AMD_PSTATE_DISABLE; + current_pstate_driver = NULL; +} + +static int amd_pstate_set_driver(int mode_idx) +{ + if (mode_idx >= AMD_PSTATE_DISABLE && mode_idx < AMD_PSTATE_MAX) { + cppc_state = mode_idx; + if (cppc_state == AMD_PSTATE_DISABLE) + pr_info("driver is explicitly disabled\n"); + + if (cppc_state == AMD_PSTATE_ACTIVE) + current_pstate_driver = &amd_pstate_epp_driver; + + if (cppc_state == AMD_PSTATE_PASSIVE || cppc_state == AMD_PSTATE_GUIDED) + current_pstate_driver = &amd_pstate_driver; + + return 0; + } + + return -EINVAL; +} + +static int amd_pstate_register_driver(int mode) +{ + int ret; + + ret = amd_pstate_set_driver(mode); + if (ret) + return ret; + + cppc_state = mode; + + /* at least one CPU supports CPB */ + current_pstate_driver->boost_enabled = cpu_feature_enabled(X86_FEATURE_CPB); + + ret = cpufreq_register_driver(current_pstate_driver); + if (ret) { + amd_pstate_driver_cleanup(); + return ret; + } + + /* Enable ITMT support once all CPUs have initialized their asym priorities. */ + if (amd_pstate_prefcore) + sched_set_itmt_support(); + + return 0; +} + +static int amd_pstate_unregister_driver(int dummy) +{ + cpufreq_unregister_driver(current_pstate_driver); + amd_pstate_driver_cleanup(); + return 0; +} + +static int amd_pstate_change_mode_without_dvr_change(int mode) +{ + int cpu = 0; + + cppc_state = mode; + + if (cpu_feature_enabled(X86_FEATURE_CPPC) || cppc_state == AMD_PSTATE_ACTIVE) + return 0; + + for_each_online_cpu(cpu) { + cppc_set_auto_sel(cpu, (cppc_state == AMD_PSTATE_PASSIVE) ? 0 : 1); + } + + return 0; +} + +static int amd_pstate_change_driver_mode(int mode) +{ + int ret; + + ret = amd_pstate_unregister_driver(0); + if (ret) + return ret; + + ret = amd_pstate_register_driver(mode); + if (ret) + return ret; + + return 0; +} + +static cppc_mode_transition_fn mode_state_machine[AMD_PSTATE_MAX][AMD_PSTATE_MAX] = { + [AMD_PSTATE_DISABLE] = { + [AMD_PSTATE_DISABLE] = NULL, + [AMD_PSTATE_PASSIVE] = amd_pstate_register_driver, + [AMD_PSTATE_ACTIVE] = amd_pstate_register_driver, + [AMD_PSTATE_GUIDED] = amd_pstate_register_driver, + }, + [AMD_PSTATE_PASSIVE] = { + [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver, + [AMD_PSTATE_PASSIVE] = NULL, + [AMD_PSTATE_ACTIVE] = amd_pstate_change_driver_mode, + [AMD_PSTATE_GUIDED] = amd_pstate_change_mode_without_dvr_change, + }, + [AMD_PSTATE_ACTIVE] = { + [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver, + [AMD_PSTATE_PASSIVE] = amd_pstate_change_driver_mode, + [AMD_PSTATE_ACTIVE] = NULL, + [AMD_PSTATE_GUIDED] = amd_pstate_change_driver_mode, + }, + [AMD_PSTATE_GUIDED] = { + [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver, + [AMD_PSTATE_PASSIVE] = amd_pstate_change_mode_without_dvr_change, + [AMD_PSTATE_ACTIVE] = amd_pstate_change_driver_mode, + [AMD_PSTATE_GUIDED] = NULL, + }, +}; + +static ssize_t amd_pstate_show_status(char *buf) +{ + if (!current_pstate_driver) + return sysfs_emit(buf, "disable\n"); + + return sysfs_emit(buf, "%s\n", amd_pstate_mode_string[cppc_state]); +} + +int amd_pstate_get_status(void) +{ + return cppc_state; +} +EXPORT_SYMBOL_GPL(amd_pstate_get_status); + +int amd_pstate_update_status(const char *buf, size_t size) +{ + int mode_idx; + + if (size > strlen("passive") || size < strlen("active")) + return -EINVAL; + + mode_idx = get_mode_idx_from_str(buf, size); + if (mode_idx < 0) + return mode_idx; + + if (mode_state_machine[cppc_state][mode_idx]) { + guard(mutex)(&amd_pstate_driver_lock); + return mode_state_machine[cppc_state][mode_idx](mode_idx); + } + + return 0; +} +EXPORT_SYMBOL_GPL(amd_pstate_update_status); + +static ssize_t status_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + + guard(mutex)(&amd_pstate_driver_lock); + + return amd_pstate_show_status(buf); +} + +static ssize_t status_store(struct device *a, struct device_attribute *b, + const char *buf, size_t count) +{ + char *p = memchr(buf, '\n', count); + int ret; + + ret = amd_pstate_update_status(buf, p ? p - buf : count); + + return ret < 0 ? ret : count; +} + +static ssize_t prefcore_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%s\n", str_enabled_disabled(amd_pstate_prefcore)); +} + +cpufreq_freq_attr_ro(amd_pstate_max_freq); +cpufreq_freq_attr_ro(amd_pstate_lowest_nonlinear_freq); + +cpufreq_freq_attr_ro(amd_pstate_highest_perf); +cpufreq_freq_attr_ro(amd_pstate_prefcore_ranking); +cpufreq_freq_attr_ro(amd_pstate_hw_prefcore); +cpufreq_freq_attr_rw(energy_performance_preference); +cpufreq_freq_attr_ro(energy_performance_available_preferences); +static DEVICE_ATTR_RW(status); +static DEVICE_ATTR_RO(prefcore); + +static struct freq_attr *amd_pstate_attr[] = { + &amd_pstate_max_freq, + &amd_pstate_lowest_nonlinear_freq, + &amd_pstate_highest_perf, + &amd_pstate_prefcore_ranking, + &amd_pstate_hw_prefcore, + NULL, +}; + +static struct freq_attr *amd_pstate_epp_attr[] = { + &amd_pstate_max_freq, + &amd_pstate_lowest_nonlinear_freq, + &amd_pstate_highest_perf, + &amd_pstate_prefcore_ranking, + &amd_pstate_hw_prefcore, + &energy_performance_preference, + &energy_performance_available_preferences, + NULL, +}; + +static struct attribute *pstate_global_attributes[] = { + &dev_attr_status.attr, + &dev_attr_prefcore.attr, + NULL +}; + +static const struct attribute_group amd_pstate_global_attr_group = { + .name = "amd_pstate", + .attrs = pstate_global_attributes, +}; + +static bool amd_pstate_acpi_pm_profile_server(void) +{ + switch (acpi_gbl_FADT.preferred_profile) { + case PM_ENTERPRISE_SERVER: + case PM_SOHO_SERVER: + case PM_PERFORMANCE_SERVER: + return true; + } + return false; +} + +static bool amd_pstate_acpi_pm_profile_undefined(void) +{ + if (acpi_gbl_FADT.preferred_profile == PM_UNSPECIFIED) + return true; + if (acpi_gbl_FADT.preferred_profile >= NR_PM_PROFILES) + return true; + return false; +} + +static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) +{ + struct amd_cpudata *cpudata; + union perf_cached perf; + struct device *dev; + int ret; + + /* + * Resetting PERF_CTL_MSR will put the CPU in P0 frequency, + * which is ideal for initialization process. + */ + amd_perf_ctl_reset(policy->cpu); + dev = get_cpu_device(policy->cpu); + if (!dev) + return -ENODEV; + + cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL); + if (!cpudata) + return -ENOMEM; + + cpudata->cpu = policy->cpu; + + ret = amd_pstate_init_perf(cpudata); + if (ret) + goto free_cpudata1; + + amd_pstate_init_prefcore(cpudata); + + ret = amd_pstate_init_freq(cpudata); + if (ret) + goto free_cpudata1; + + ret = amd_pstate_init_boost_support(cpudata); + if (ret) + goto free_cpudata1; + + perf = READ_ONCE(cpudata->perf); + + policy->cpuinfo.min_freq = policy->min = perf_to_freq(perf, + cpudata->nominal_freq, + perf.lowest_perf); + policy->cpuinfo.max_freq = policy->max = perf_to_freq(perf, + cpudata->nominal_freq, + perf.highest_perf); + policy->driver_data = cpudata; + + ret = amd_pstate_cppc_enable(policy); + if (ret) + goto free_cpudata1; + + /* It will be updated by governor */ + policy->cur = policy->cpuinfo.min_freq; + + + policy->boost_supported = READ_ONCE(cpudata->boost_supported); + + /* + * Set the policy to provide a valid fallback value in case + * the default cpufreq governor is neither powersave nor performance. + */ + if (amd_pstate_acpi_pm_profile_server() || + amd_pstate_acpi_pm_profile_undefined()) { + policy->policy = CPUFREQ_POLICY_PERFORMANCE; + cpudata->epp_default = amd_pstate_get_epp(cpudata); + } else { + policy->policy = CPUFREQ_POLICY_POWERSAVE; + cpudata->epp_default = AMD_CPPC_EPP_BALANCE_PERFORMANCE; + } + + ret = amd_pstate_set_epp(policy, cpudata->epp_default); + if (ret) + return ret; + + current_pstate_driver->adjust_perf = NULL; + + return 0; + +free_cpudata1: + pr_warn("Failed to initialize CPU %d: %d\n", policy->cpu, ret); + kfree(cpudata); + return ret; +} + +static void amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy) +{ + struct amd_cpudata *cpudata = policy->driver_data; + + if (cpudata) { + union perf_cached perf = READ_ONCE(cpudata->perf); + + /* Reset CPPC_REQ MSR to the BIOS value */ + amd_pstate_update_perf(policy, perf.bios_min_perf, 0U, 0U, 0U, false); + + kfree(cpudata); + policy->driver_data = NULL; + } + + pr_debug("CPU %d exiting\n", policy->cpu); +} + +static int amd_pstate_epp_update_limit(struct cpufreq_policy *policy, bool policy_change) +{ + struct amd_cpudata *cpudata = policy->driver_data; + union perf_cached perf; + u8 epp; + + if (policy_change || + policy->min != cpudata->min_limit_freq || + policy->max != cpudata->max_limit_freq) + amd_pstate_update_min_max_limit(policy); + + if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) + epp = 0; + else + epp = FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached); + + perf = READ_ONCE(cpudata->perf); + + return amd_pstate_update_perf(policy, perf.min_limit_perf, 0U, + perf.max_limit_perf, epp, false); +} + +static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy) +{ + struct amd_cpudata *cpudata = policy->driver_data; + int ret; + + if (!policy->cpuinfo.max_freq) + return -ENODEV; + + cpudata->policy = policy->policy; + + ret = amd_pstate_epp_update_limit(policy, true); + if (ret) + return ret; + + /* + * policy->cur is never updated with the amd_pstate_epp driver, but it + * is used as a stale frequency value. So, keep it within limits. + */ + policy->cur = policy->min; + + return 0; +} + +static int amd_pstate_cpu_online(struct cpufreq_policy *policy) +{ + return amd_pstate_cppc_enable(policy); +} + +static int amd_pstate_cpu_offline(struct cpufreq_policy *policy) +{ + struct amd_cpudata *cpudata = policy->driver_data; + union perf_cached perf = READ_ONCE(cpudata->perf); + + /* + * Reset CPPC_REQ MSR to the BIOS value, this will allow us to retain the BIOS specified + * min_perf value across kexec reboots. If this CPU is just onlined normally after this, the + * limits, epp and desired perf will get reset to the cached values in cpudata struct + */ + return amd_pstate_update_perf(policy, perf.bios_min_perf, + FIELD_GET(AMD_CPPC_DES_PERF_MASK, cpudata->cppc_req_cached), + FIELD_GET(AMD_CPPC_MAX_PERF_MASK, cpudata->cppc_req_cached), + FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached), + false); +} + +static int amd_pstate_suspend(struct cpufreq_policy *policy) +{ + struct amd_cpudata *cpudata = policy->driver_data; + union perf_cached perf = READ_ONCE(cpudata->perf); + int ret; + + /* + * Reset CPPC_REQ MSR to the BIOS value, this will allow us to retain the BIOS specified + * min_perf value across kexec reboots. If this CPU is just resumed back without kexec, + * the limits, epp and desired perf will get reset to the cached values in cpudata struct + */ + ret = amd_pstate_update_perf(policy, perf.bios_min_perf, + FIELD_GET(AMD_CPPC_DES_PERF_MASK, cpudata->cppc_req_cached), + FIELD_GET(AMD_CPPC_MAX_PERF_MASK, cpudata->cppc_req_cached), + FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached), + false); + if (ret) + return ret; + + /* set this flag to avoid setting core offline*/ + cpudata->suspended = true; + + return 0; +} + +static int amd_pstate_resume(struct cpufreq_policy *policy) +{ + struct amd_cpudata *cpudata = policy->driver_data; + union perf_cached perf = READ_ONCE(cpudata->perf); + int cur_perf = freq_to_perf(perf, cpudata->nominal_freq, policy->cur); + + /* Set CPPC_REQ to last sane value until the governor updates it */ + return amd_pstate_update_perf(policy, perf.min_limit_perf, cur_perf, perf.max_limit_perf, + 0U, false); +} + +static int amd_pstate_epp_resume(struct cpufreq_policy *policy) +{ + struct amd_cpudata *cpudata = policy->driver_data; + + if (cpudata->suspended) { + int ret; + + /* enable amd pstate from suspend state*/ + ret = amd_pstate_epp_update_limit(policy, false); + if (ret) + return ret; + + cpudata->suspended = false; + } + + return 0; +} + +static struct cpufreq_driver amd_pstate_driver = { + .flags = CPUFREQ_CONST_LOOPS | CPUFREQ_NEED_UPDATE_LIMITS, + .verify = amd_pstate_verify, + .target = amd_pstate_target, + .fast_switch = amd_pstate_fast_switch, + .init = amd_pstate_cpu_init, + .exit = amd_pstate_cpu_exit, + .online = amd_pstate_cpu_online, + .offline = amd_pstate_cpu_offline, + .suspend = amd_pstate_suspend, + .resume = amd_pstate_resume, + .set_boost = amd_pstate_set_boost, + .update_limits = amd_pstate_update_limits, + .name = "amd-pstate", + .attr = amd_pstate_attr, +}; + +static struct cpufreq_driver amd_pstate_epp_driver = { + .flags = CPUFREQ_CONST_LOOPS, + .verify = amd_pstate_verify, + .setpolicy = amd_pstate_epp_set_policy, + .init = amd_pstate_epp_cpu_init, + .exit = amd_pstate_epp_cpu_exit, + .offline = amd_pstate_cpu_offline, + .online = amd_pstate_cpu_online, + .suspend = amd_pstate_suspend, + .resume = amd_pstate_epp_resume, + .update_limits = amd_pstate_update_limits, + .set_boost = amd_pstate_set_boost, + .name = "amd-pstate-epp", + .attr = amd_pstate_epp_attr, +}; + +/* + * CPPC function is not supported for family ID 17H with model_ID ranging from 0x10 to 0x2F. + * show the debug message that helps to check if the CPU has CPPC support for loading issue. + */ +static bool amd_cppc_supported(void) +{ + struct cpuinfo_x86 *c = &cpu_data(0); + bool warn = false; + + if ((boot_cpu_data.x86 == 0x17) && (boot_cpu_data.x86_model < 0x30)) { + pr_debug_once("CPPC feature is not supported by the processor\n"); + return false; + } + + /* + * If the CPPC feature is disabled in the BIOS for processors + * that support MSR-based CPPC, the AMD Pstate driver may not + * function correctly. + * + * For such processors, check the CPPC flag and display a + * warning message if the platform supports CPPC. + * + * Note: The code check below will not abort the driver + * registration process because of the code is added for + * debugging purposes. Besides, it may still be possible for + * the driver to work using the shared-memory mechanism. + */ + if (!cpu_feature_enabled(X86_FEATURE_CPPC)) { + if (cpu_feature_enabled(X86_FEATURE_ZEN2)) { + switch (c->x86_model) { + case 0x60 ... 0x6F: + case 0x80 ... 0xAF: + warn = true; + break; + } + } else if (cpu_feature_enabled(X86_FEATURE_ZEN3) || + cpu_feature_enabled(X86_FEATURE_ZEN4)) { + switch (c->x86_model) { + case 0x10 ... 0x1F: + case 0x40 ... 0xAF: + warn = true; + break; + } + } else if (cpu_feature_enabled(X86_FEATURE_ZEN5)) { + warn = true; + } + } + + if (warn) + pr_warn_once("The CPPC feature is supported but currently disabled by the BIOS.\n" + "Please enable it if your BIOS has the CPPC option.\n"); + return true; +} + +static int __init amd_pstate_init(void) +{ + struct device *dev_root; + int ret; + + if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD) + return -ENODEV; + + /* show debug message only if CPPC is not supported */ + if (!amd_cppc_supported()) + return -EOPNOTSUPP; + + /* show warning message when BIOS broken or ACPI disabled */ + if (!acpi_cpc_valid()) { + pr_warn_once("the _CPC object is not present in SBIOS or ACPI disabled\n"); + return -ENODEV; + } + + /* don't keep reloading if cpufreq_driver exists */ + if (cpufreq_get_current_driver()) + return -EEXIST; + + quirks = NULL; + + /* check if this machine need CPPC quirks */ + dmi_check_system(amd_pstate_quirks_table); + + /* + * determine the driver mode from the command line or kernel config. + * If no command line input is provided, cppc_state will be AMD_PSTATE_UNDEFINED. + * command line options will override the kernel config settings. + */ + + if (cppc_state == AMD_PSTATE_UNDEFINED) { + /* Disable on the following configs by default: + * 1. Undefined platforms + * 2. Server platforms with CPUs older than Family 0x1A. + */ + if (amd_pstate_acpi_pm_profile_undefined() || + (amd_pstate_acpi_pm_profile_server() && boot_cpu_data.x86 < 0x1A)) { + pr_info("driver load is disabled, boot with specific mode to enable this\n"); + return -ENODEV; + } + /* get driver mode from kernel config option [1:4] */ + cppc_state = CONFIG_X86_AMD_PSTATE_DEFAULT_MODE; + } + + if (cppc_state == AMD_PSTATE_DISABLE) { + pr_info("driver load is disabled, boot with specific mode to enable this\n"); + return -ENODEV; + } + + /* capability check */ + if (cpu_feature_enabled(X86_FEATURE_CPPC)) { + pr_debug("AMD CPPC MSR based functionality is supported\n"); + } else { + pr_debug("AMD CPPC shared memory based functionality is supported\n"); + static_call_update(amd_pstate_cppc_enable, shmem_cppc_enable); + static_call_update(amd_pstate_init_perf, shmem_init_perf); + static_call_update(amd_pstate_update_perf, shmem_update_perf); + static_call_update(amd_pstate_get_epp, shmem_get_epp); + static_call_update(amd_pstate_set_epp, shmem_set_epp); + } + + if (amd_pstate_prefcore) { + ret = amd_detect_prefcore(&amd_pstate_prefcore); + if (ret) + return ret; + } + + ret = amd_pstate_register_driver(cppc_state); + if (ret) { + pr_err("failed to register with return %d\n", ret); + return ret; + } + + dev_root = bus_get_dev_root(&cpu_subsys); + if (dev_root) { + ret = sysfs_create_group(&dev_root->kobj, &amd_pstate_global_attr_group); + put_device(dev_root); + if (ret) { + pr_err("sysfs attribute export failed with error %d.\n", ret); + goto global_attr_free; + } + } + + return ret; + +global_attr_free: + cpufreq_unregister_driver(current_pstate_driver); + return ret; +} +device_initcall(amd_pstate_init); + +static int __init amd_pstate_param(char *str) +{ + size_t size; + int mode_idx; + + if (!str) + return -EINVAL; + + size = strlen(str); + mode_idx = get_mode_idx_from_str(str, size); + + return amd_pstate_set_driver(mode_idx); +} + +static int __init amd_prefcore_param(char *str) +{ + if (!strcmp(str, "disable")) + amd_pstate_prefcore = false; + + return 0; +} + +early_param("amd_pstate", amd_pstate_param); +early_param("amd_prefcore", amd_prefcore_param); + +MODULE_AUTHOR("Huang Rui <ray.huang@amd.com>"); +MODULE_DESCRIPTION("AMD Processor P-state Frequency Driver"); diff --git a/drivers/cpufreq/amd-pstate.h b/drivers/cpufreq/amd-pstate.h new file mode 100644 index 000000000000..cb45fdca27a6 --- /dev/null +++ b/drivers/cpufreq/amd-pstate.h @@ -0,0 +1,127 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2022 Advanced Micro Devices, Inc. + * + * Author: Meng Li <li.meng@amd.com> + */ + +#ifndef _LINUX_AMD_PSTATE_H +#define _LINUX_AMD_PSTATE_H + +#include <linux/pm_qos.h> + +/********************************************************************* + * AMD P-state INTERFACE * + *********************************************************************/ + +/** + * union perf_cached - A union to cache performance-related data. + * @highest_perf: the maximum performance an individual processor may reach, + * assuming ideal conditions + * For platforms that support the preferred core feature, the highest_perf value maybe + * configured to any value in the range 166-255 by the firmware (because the preferred + * core ranking is encoded in the highest_perf value). To maintain consistency across + * all platforms, we split the highest_perf and preferred core ranking values into + * cpudata->perf.highest_perf and cpudata->prefcore_ranking. + * @nominal_perf: the maximum sustained performance level of the processor, + * assuming ideal operating conditions + * @lowest_nonlinear_perf: the lowest performance level at which nonlinear power + * savings are achieved + * @lowest_perf: the absolute lowest performance level of the processor + * @min_limit_perf: Cached value of the performance corresponding to policy->min + * @max_limit_perf: Cached value of the performance corresponding to policy->max + * @bios_min_perf: Cached perf value corresponding to the "Requested CPU Min Frequency" BIOS option + */ +union perf_cached { + struct { + u8 highest_perf; + u8 nominal_perf; + u8 lowest_nonlinear_perf; + u8 lowest_perf; + u8 min_limit_perf; + u8 max_limit_perf; + u8 bios_min_perf; + }; + u64 val; +}; + +/** + * struct amd_aperf_mperf + * @aperf: actual performance frequency clock count + * @mperf: maximum performance frequency clock count + * @tsc: time stamp counter + */ +struct amd_aperf_mperf { + u64 aperf; + u64 mperf; + u64 tsc; +}; + +/** + * struct amd_cpudata - private CPU data for AMD P-State + * @cpu: CPU number + * @req: constraint request to apply + * @cppc_req_cached: cached performance request hints + * @perf: cached performance-related data + * @prefcore_ranking: the preferred core ranking, the higher value indicates a higher + * priority. + * @min_limit_freq: Cached value of policy->min (in khz) + * @max_limit_freq: Cached value of policy->max (in khz) + * @nominal_freq: the frequency (in khz) that mapped to nominal_perf + * @lowest_nonlinear_freq: the frequency (in khz) that mapped to lowest_nonlinear_perf + * @cur: Difference of Aperf/Mperf/tsc count between last and current sample + * @prev: Last Aperf/Mperf/tsc count value read from register + * @freq: current cpu frequency value (in khz) + * @boost_supported: check whether the Processor or SBIOS supports boost mode + * @hw_prefcore: check whether HW supports preferred core featue. + * Only when hw_prefcore and early prefcore param are true, + * AMD P-State driver supports preferred core featue. + * @epp_cached: Cached CPPC energy-performance preference value + * @policy: Cpufreq policy value + * + * The amd_cpudata is key private data for each CPU thread in AMD P-State, and + * represents all the attributes and goals that AMD P-State requests at runtime. + */ +struct amd_cpudata { + int cpu; + + struct freq_qos_request req[2]; + u64 cppc_req_cached; + + union perf_cached perf; + + u8 prefcore_ranking; + u32 min_limit_freq; + u32 max_limit_freq; + u32 nominal_freq; + u32 lowest_nonlinear_freq; + + struct amd_aperf_mperf cur; + struct amd_aperf_mperf prev; + + u64 freq; + bool boost_supported; + bool hw_prefcore; + + /* EPP feature related attributes*/ + u32 policy; + bool suspended; + u8 epp_default; +}; + +/* + * enum amd_pstate_mode - driver working mode of amd pstate + */ +enum amd_pstate_mode { + AMD_PSTATE_UNDEFINED = 0, + AMD_PSTATE_DISABLE, + AMD_PSTATE_PASSIVE, + AMD_PSTATE_ACTIVE, + AMD_PSTATE_GUIDED, + AMD_PSTATE_MAX, +}; +const char *amd_pstate_get_mode_string(enum amd_pstate_mode mode); +int amd_pstate_get_status(void); +int amd_pstate_update_status(const char *buf, size_t size); + +#endif /* _LINUX_AMD_PSTATE_H */ diff --git a/drivers/cpufreq/amd_freq_sensitivity.c b/drivers/cpufreq/amd_freq_sensitivity.c index f6b79ab0070b..13fed4b9e02b 100644 --- a/drivers/cpufreq/amd_freq_sensitivity.c +++ b/drivers/cpufreq/amd_freq_sensitivity.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * amd_freq_sensitivity.c: AMD frequency sensitivity feedback powersave bias * for the ondemand governor. @@ -5,23 +6,21 @@ * Copyright (C) 2013 Advanced Micro Devices, Inc. * * Author: Jacob Shin <jacob.shin@amd.com> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #include <linux/kernel.h> #include <linux/module.h> #include <linux/types.h> +#include <linux/pci.h> #include <linux/percpu-defs.h> #include <linux/init.h> #include <linux/mod_devicetable.h> #include <asm/msr.h> #include <asm/cpufeature.h> +#include <asm/cpu_device_id.h> -#include "cpufreq_governor.h" +#include "cpufreq_ondemand.h" #define MSR_AMD64_FREQ_SENSITIVITY_ACTUAL 0xc0010080 #define MSR_AMD64_FREQ_SENSITIVITY_REFERENCE 0xc0010081 @@ -45,12 +44,11 @@ static unsigned int amd_powersave_bias_target(struct cpufreq_policy *policy, long d_actual, d_reference; struct msr actual, reference; struct cpu_data_t *data = &per_cpu(cpu_data, policy->cpu); - struct dbs_data *od_data = policy->governor_data; + struct policy_dbs_info *policy_dbs = policy->governor_data; + struct dbs_data *od_data = policy_dbs->dbs_data; struct od_dbs_tuners *od_tuners = od_data->tuners; - struct od_cpu_dbs_info_s *od_info = - od_data->cdata->get_cpu_dbs_info_s(policy->cpu); - if (!od_info->freq_table) + if (!policy->freq_table) return freq_next; rdmsr_on_cpu(policy->cpu, MSR_AMD64_FREQ_SENSITIVITY_ACTUAL, @@ -92,10 +90,10 @@ static unsigned int amd_powersave_bias_target(struct cpufreq_policy *policy, else { unsigned int index; - cpufreq_frequency_table_target(policy, - od_info->freq_table, policy->cur - 1, - CPUFREQ_RELATION_H, &index); - freq_next = od_info->freq_table[index].frequency; + index = cpufreq_table_find_index_h(policy, + policy->cur - 1, + relation & CPUFREQ_RELATION_E); + freq_next = policy->freq_table[index].frequency; } data->freq_prev = freq_next; @@ -111,14 +109,27 @@ out: static int __init amd_freq_sensitivity_init(void) { u64 val; - - if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD) + struct pci_dev *pcidev; + unsigned int pci_vendor; + + if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) + pci_vendor = PCI_VENDOR_ID_AMD; + else if (boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) + pci_vendor = PCI_VENDOR_ID_HYGON; + else return -ENODEV; - if (!static_cpu_has(X86_FEATURE_PROC_FEEDBACK)) - return -ENODEV; + pcidev = pci_get_device(pci_vendor, + PCI_DEVICE_ID_AMD_KERNCZ_SMBUS, NULL); + + if (!pcidev) { + if (!boot_cpu_has(X86_FEATURE_PROC_FEEDBACK)) + return -ENODEV; + } else { + pci_dev_put(pcidev); + } - if (rdmsrl_safe(MSR_AMD64_FREQ_SENSITIVITY_ACTUAL, &val)) + if (rdmsrq_safe(MSR_AMD64_FREQ_SENSITIVITY_ACTUAL, &val)) return -ENODEV; if (!(val >> CLASS_CODE_SHIFT)) @@ -136,8 +147,8 @@ static void __exit amd_freq_sensitivity_exit(void) } module_exit(amd_freq_sensitivity_exit); -static const struct x86_cpu_id amd_freq_sensitivity_ids[] = { - X86_FEATURE_MATCH(X86_FEATURE_PROC_FEEDBACK), +static const struct x86_cpu_id __maybe_unused amd_freq_sensitivity_ids[] = { + X86_MATCH_FEATURE(X86_FEATURE_PROC_FEEDBACK, NULL), {} }; MODULE_DEVICE_TABLE(x86cpu, amd_freq_sensitivity_ids); diff --git a/drivers/cpufreq/apple-soc-cpufreq.c b/drivers/cpufreq/apple-soc-cpufreq.c new file mode 100644 index 000000000000..b1d29b7af232 --- /dev/null +++ b/drivers/cpufreq/apple-soc-cpufreq.c @@ -0,0 +1,375 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Apple SoC CPU cluster performance state driver + * + * Copyright The Asahi Linux Contributors + * + * Based on scpi-cpufreq.c + */ + +#include <linux/bitfield.h> +#include <linux/bitops.h> +#include <linux/cpu.h> +#include <linux/cpufreq.h> +#include <linux/cpumask.h> +#include <linux/delay.h> +#include <linux/err.h> +#include <linux/io.h> +#include <linux/iopoll.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_address.h> +#include <linux/pm_opp.h> +#include <linux/slab.h> + +#define APPLE_DVFS_CMD 0x20 +#define APPLE_DVFS_CMD_BUSY BIT(31) +#define APPLE_DVFS_CMD_SET BIT(25) +#define APPLE_DVFS_CMD_PS1_S5L8960X GENMASK(24, 22) +#define APPLE_DVFS_CMD_PS1_S5L8960X_SHIFT 22 +#define APPLE_DVFS_CMD_PS2 GENMASK(15, 12) +#define APPLE_DVFS_CMD_PS1 GENMASK(4, 0) +#define APPLE_DVFS_CMD_PS1_SHIFT 0 + +/* Same timebase as CPU counter (24MHz) */ +#define APPLE_DVFS_LAST_CHG_TIME 0x38 + +/* + * Apple ran out of bits and had to shift this in T8112... + */ +#define APPLE_DVFS_STATUS 0x50 +#define APPLE_DVFS_STATUS_CUR_PS_S5L8960X GENMASK(5, 3) +#define APPLE_DVFS_STATUS_CUR_PS_SHIFT_S5L8960X 3 +#define APPLE_DVFS_STATUS_TGT_PS_S5L8960X GENMASK(2, 0) +#define APPLE_DVFS_STATUS_CUR_PS_T8103 GENMASK(7, 4) +#define APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8103 4 +#define APPLE_DVFS_STATUS_TGT_PS_T8103 GENMASK(3, 0) +#define APPLE_DVFS_STATUS_CUR_PS_T8112 GENMASK(9, 5) +#define APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8112 5 +#define APPLE_DVFS_STATUS_TGT_PS_T8112 GENMASK(4, 0) + +/* + * Div is +1, base clock is 12MHz on existing SoCs. + * For documentation purposes. We use the OPP table to + * get the frequency. + */ +#define APPLE_DVFS_PLL_STATUS 0xc0 +#define APPLE_DVFS_PLL_FACTOR 0xc8 +#define APPLE_DVFS_PLL_FACTOR_MULT GENMASK(31, 16) +#define APPLE_DVFS_PLL_FACTOR_DIV GENMASK(15, 0) + +#define APPLE_DVFS_TRANSITION_TIMEOUT 400 + +struct apple_soc_cpufreq_info { + bool has_ps2; + u64 max_pstate; + u64 cur_pstate_mask; + u64 cur_pstate_shift; + u64 ps1_mask; + u64 ps1_shift; +}; + +struct apple_cpu_priv { + struct device *cpu_dev; + void __iomem *reg_base; + const struct apple_soc_cpufreq_info *info; +}; + +static struct cpufreq_driver apple_soc_cpufreq_driver; + +static const struct apple_soc_cpufreq_info soc_s5l8960x_info = { + .has_ps2 = false, + .max_pstate = 7, + .cur_pstate_mask = APPLE_DVFS_STATUS_CUR_PS_S5L8960X, + .cur_pstate_shift = APPLE_DVFS_STATUS_CUR_PS_SHIFT_S5L8960X, + .ps1_mask = APPLE_DVFS_CMD_PS1_S5L8960X, + .ps1_shift = APPLE_DVFS_CMD_PS1_S5L8960X_SHIFT, +}; + +static const struct apple_soc_cpufreq_info soc_t8103_info = { + .has_ps2 = true, + .max_pstate = 15, + .cur_pstate_mask = APPLE_DVFS_STATUS_CUR_PS_T8103, + .cur_pstate_shift = APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8103, + .ps1_mask = APPLE_DVFS_CMD_PS1, + .ps1_shift = APPLE_DVFS_CMD_PS1_SHIFT, +}; + +static const struct apple_soc_cpufreq_info soc_t8112_info = { + .has_ps2 = false, + .max_pstate = 31, + .cur_pstate_mask = APPLE_DVFS_STATUS_CUR_PS_T8112, + .cur_pstate_shift = APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8112, + .ps1_mask = APPLE_DVFS_CMD_PS1, + .ps1_shift = APPLE_DVFS_CMD_PS1_SHIFT, +}; + +static const struct apple_soc_cpufreq_info soc_default_info = { + .has_ps2 = false, + .max_pstate = 15, + .cur_pstate_mask = 0, /* fallback */ + .ps1_mask = APPLE_DVFS_CMD_PS1, + .ps1_shift = APPLE_DVFS_CMD_PS1_SHIFT, +}; + +static const struct of_device_id apple_soc_cpufreq_of_match[] __maybe_unused = { + { + .compatible = "apple,s5l8960x-cluster-cpufreq", + .data = &soc_s5l8960x_info, + }, + { + .compatible = "apple,t8103-cluster-cpufreq", + .data = &soc_t8103_info, + }, + { + .compatible = "apple,t8112-cluster-cpufreq", + .data = &soc_t8112_info, + }, + { + .compatible = "apple,cluster-cpufreq", + .data = &soc_default_info, + }, + {} +}; + +static unsigned int apple_soc_cpufreq_get_rate(unsigned int cpu) +{ + struct cpufreq_policy *policy; + struct apple_cpu_priv *priv; + struct cpufreq_frequency_table *p; + unsigned int pstate; + + policy = cpufreq_cpu_get_raw(cpu); + if (unlikely(!policy)) + return 0; + + priv = policy->driver_data; + + if (priv->info->cur_pstate_mask) { + u32 reg = readl_relaxed(priv->reg_base + APPLE_DVFS_STATUS); + + pstate = (reg & priv->info->cur_pstate_mask) >> priv->info->cur_pstate_shift; + } else { + /* + * For the fallback case we might not know the layout of DVFS_STATUS, + * so just use the command register value (which ignores boost limitations). + */ + u64 reg = readq_relaxed(priv->reg_base + APPLE_DVFS_CMD); + + pstate = FIELD_GET(APPLE_DVFS_CMD_PS1, reg); + } + + cpufreq_for_each_valid_entry(p, policy->freq_table) + if (p->driver_data == pstate) + return p->frequency; + + dev_err(priv->cpu_dev, "could not find frequency for pstate %d\n", + pstate); + return 0; +} + +static int apple_soc_cpufreq_set_target(struct cpufreq_policy *policy, + unsigned int index) +{ + struct apple_cpu_priv *priv = policy->driver_data; + unsigned int pstate = policy->freq_table[index].driver_data; + u64 reg; + + /* Fallback for newer SoCs */ + if (index > priv->info->max_pstate) + index = priv->info->max_pstate; + + if (readq_poll_timeout_atomic(priv->reg_base + APPLE_DVFS_CMD, reg, + !(reg & APPLE_DVFS_CMD_BUSY), 2, + APPLE_DVFS_TRANSITION_TIMEOUT)) { + return -EIO; + } + + reg &= ~priv->info->ps1_mask; + reg |= pstate << priv->info->ps1_shift; + if (priv->info->has_ps2) { + reg &= ~APPLE_DVFS_CMD_PS2; + reg |= FIELD_PREP(APPLE_DVFS_CMD_PS2, pstate); + } + reg |= APPLE_DVFS_CMD_SET; + + writeq_relaxed(reg, priv->reg_base + APPLE_DVFS_CMD); + + return 0; +} + +static unsigned int apple_soc_cpufreq_fast_switch(struct cpufreq_policy *policy, + unsigned int target_freq) +{ + if (apple_soc_cpufreq_set_target(policy, policy->cached_resolved_idx) < 0) + return 0; + + return policy->freq_table[policy->cached_resolved_idx].frequency; +} + +static int apple_soc_cpufreq_find_cluster(struct cpufreq_policy *policy, + void __iomem **reg_base, + const struct apple_soc_cpufreq_info **info) +{ + struct of_phandle_args args; + const struct of_device_id *match; + int ret = 0; + + ret = of_perf_domain_get_sharing_cpumask(policy->cpu, "performance-domains", + "#performance-domain-cells", + policy->cpus, &args); + if (ret < 0) + return ret; + + match = of_match_node(apple_soc_cpufreq_of_match, args.np); + of_node_put(args.np); + if (!match) + return -ENODEV; + + *info = match->data; + + *reg_base = of_iomap(args.np, 0); + if (!*reg_base) + return -ENOMEM; + + return 0; +} + +static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) +{ + int ret, i; + unsigned int transition_latency; + void __iomem *reg_base; + struct device *cpu_dev; + struct apple_cpu_priv *priv; + const struct apple_soc_cpufreq_info *info; + struct cpufreq_frequency_table *freq_table; + + cpu_dev = get_cpu_device(policy->cpu); + if (!cpu_dev) { + pr_err("failed to get cpu%d device\n", policy->cpu); + return -ENODEV; + } + + ret = dev_pm_opp_of_add_table(cpu_dev); + if (ret < 0) { + dev_err(cpu_dev, "%s: failed to add OPP table: %d\n", __func__, ret); + return ret; + } + + ret = apple_soc_cpufreq_find_cluster(policy, ®_base, &info); + if (ret) { + dev_err(cpu_dev, "%s: failed to get cluster info: %d\n", __func__, ret); + return ret; + } + + ret = dev_pm_opp_set_sharing_cpus(cpu_dev, policy->cpus); + if (ret) { + dev_err(cpu_dev, "%s: failed to mark OPPs as shared: %d\n", __func__, ret); + goto out_iounmap; + } + + ret = dev_pm_opp_get_opp_count(cpu_dev); + if (ret <= 0) { + dev_dbg(cpu_dev, "OPP table is not ready, deferring probe\n"); + ret = -EPROBE_DEFER; + goto out_free_opp; + } + + priv = kzalloc(sizeof(*priv), GFP_KERNEL); + if (!priv) { + ret = -ENOMEM; + goto out_free_opp; + } + + ret = dev_pm_opp_init_cpufreq_table(cpu_dev, &freq_table); + if (ret) { + dev_err(cpu_dev, "failed to init cpufreq table: %d\n", ret); + goto out_free_priv; + } + + /* Get OPP levels (p-state indexes) and stash them in driver_data */ + for (i = 0; freq_table[i].frequency != CPUFREQ_TABLE_END; i++) { + unsigned long rate = freq_table[i].frequency * 1000 + 999; + struct dev_pm_opp *opp = dev_pm_opp_find_freq_floor(cpu_dev, &rate); + + if (IS_ERR(opp)) { + ret = PTR_ERR(opp); + goto out_free_cpufreq_table; + } + freq_table[i].driver_data = dev_pm_opp_get_level(opp); + dev_pm_opp_put(opp); + } + + priv->cpu_dev = cpu_dev; + priv->reg_base = reg_base; + priv->info = info; + policy->driver_data = priv; + policy->freq_table = freq_table; + + transition_latency = dev_pm_opp_get_max_transition_latency(cpu_dev); + if (!transition_latency) + transition_latency = APPLE_DVFS_TRANSITION_TIMEOUT * NSEC_PER_USEC; + + policy->cpuinfo.transition_latency = transition_latency; + policy->dvfs_possible_from_any_cpu = true; + policy->fast_switch_possible = true; + policy->suspend_freq = freq_table[0].frequency; + + return 0; + +out_free_cpufreq_table: + dev_pm_opp_free_cpufreq_table(cpu_dev, &freq_table); +out_free_priv: + kfree(priv); +out_free_opp: + dev_pm_opp_remove_all_dynamic(cpu_dev); +out_iounmap: + iounmap(reg_base); + return ret; +} + +static void apple_soc_cpufreq_exit(struct cpufreq_policy *policy) +{ + struct apple_cpu_priv *priv = policy->driver_data; + + dev_pm_opp_free_cpufreq_table(priv->cpu_dev, &policy->freq_table); + dev_pm_opp_remove_all_dynamic(priv->cpu_dev); + iounmap(priv->reg_base); + kfree(priv); +} + +static struct cpufreq_driver apple_soc_cpufreq_driver = { + .name = "apple-cpufreq", + .flags = CPUFREQ_HAVE_GOVERNOR_PER_POLICY | + CPUFREQ_NEED_INITIAL_FREQ_CHECK | CPUFREQ_IS_COOLING_DEV, + .verify = cpufreq_generic_frequency_table_verify, + .get = apple_soc_cpufreq_get_rate, + .init = apple_soc_cpufreq_init, + .exit = apple_soc_cpufreq_exit, + .target_index = apple_soc_cpufreq_set_target, + .fast_switch = apple_soc_cpufreq_fast_switch, + .register_em = cpufreq_register_em_with_opp, + .set_boost = cpufreq_boost_set_sw, + .suspend = cpufreq_generic_suspend, +}; + +static int __init apple_soc_cpufreq_module_init(void) +{ + if (!of_machine_is_compatible("apple,arm-platform")) + return -ENODEV; + + return cpufreq_register_driver(&apple_soc_cpufreq_driver); +} +module_init(apple_soc_cpufreq_module_init); + +static void __exit apple_soc_cpufreq_module_exit(void) +{ + cpufreq_unregister_driver(&apple_soc_cpufreq_driver); +} +module_exit(apple_soc_cpufreq_module_exit); + +MODULE_DEVICE_TABLE(of, apple_soc_cpufreq_of_match); +MODULE_AUTHOR("Hector Martin <marcan@marcan.st>"); +MODULE_DESCRIPTION("Apple SoC CPU cluster DVFS driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/cpufreq/arm_big_little.c b/drivers/cpufreq/arm_big_little.c deleted file mode 100644 index 3549f0784af1..000000000000 --- a/drivers/cpufreq/arm_big_little.c +++ /dev/null @@ -1,271 +0,0 @@ -/* - * ARM big.LITTLE Platforms CPUFreq support - * - * Copyright (C) 2013 ARM Ltd. - * Sudeep KarkadaNagesha <sudeep.karkadanagesha@arm.com> - * - * Copyright (C) 2013 Linaro. - * Viresh Kumar <viresh.kumar@linaro.org> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - */ - -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - -#include <linux/clk.h> -#include <linux/cpu.h> -#include <linux/cpufreq.h> -#include <linux/cpumask.h> -#include <linux/export.h> -#include <linux/of_platform.h> -#include <linux/opp.h> -#include <linux/slab.h> -#include <linux/topology.h> -#include <linux/types.h> - -#include "arm_big_little.h" - -/* Currently we support only two clusters */ -#define MAX_CLUSTERS 2 - -static struct cpufreq_arm_bL_ops *arm_bL_ops; -static struct clk *clk[MAX_CLUSTERS]; -static struct cpufreq_frequency_table *freq_table[MAX_CLUSTERS]; -static atomic_t cluster_usage[MAX_CLUSTERS] = {ATOMIC_INIT(0), ATOMIC_INIT(0)}; - -static unsigned int bL_cpufreq_get(unsigned int cpu) -{ - u32 cur_cluster = cpu_to_cluster(cpu); - - return clk_get_rate(clk[cur_cluster]) / 1000; -} - -/* Validate policy frequency range */ -static int bL_cpufreq_verify_policy(struct cpufreq_policy *policy) -{ - u32 cur_cluster = cpu_to_cluster(policy->cpu); - - return cpufreq_frequency_table_verify(policy, freq_table[cur_cluster]); -} - -/* Set clock frequency */ -static int bL_cpufreq_set_target(struct cpufreq_policy *policy, - unsigned int target_freq, unsigned int relation) -{ - struct cpufreq_freqs freqs; - u32 cpu = policy->cpu, freq_tab_idx, cur_cluster; - int ret = 0; - - cur_cluster = cpu_to_cluster(policy->cpu); - - freqs.old = bL_cpufreq_get(policy->cpu); - - /* Determine valid target frequency using freq_table */ - cpufreq_frequency_table_target(policy, freq_table[cur_cluster], - target_freq, relation, &freq_tab_idx); - freqs.new = freq_table[cur_cluster][freq_tab_idx].frequency; - - pr_debug("%s: cpu: %d, cluster: %d, oldfreq: %d, target freq: %d, new freq: %d\n", - __func__, cpu, cur_cluster, freqs.old, target_freq, - freqs.new); - - if (freqs.old == freqs.new) - return 0; - - cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE); - - ret = clk_set_rate(clk[cur_cluster], freqs.new * 1000); - if (ret) { - pr_err("clk_set_rate failed: %d\n", ret); - freqs.new = freqs.old; - } - - cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); - - return ret; -} - -static void put_cluster_clk_and_freq_table(struct device *cpu_dev) -{ - u32 cluster = cpu_to_cluster(cpu_dev->id); - - if (!atomic_dec_return(&cluster_usage[cluster])) { - clk_put(clk[cluster]); - opp_free_cpufreq_table(cpu_dev, &freq_table[cluster]); - dev_dbg(cpu_dev, "%s: cluster: %d\n", __func__, cluster); - } -} - -static int get_cluster_clk_and_freq_table(struct device *cpu_dev) -{ - u32 cluster = cpu_to_cluster(cpu_dev->id); - char name[14] = "cpu-cluster."; - int ret; - - if (atomic_inc_return(&cluster_usage[cluster]) != 1) - return 0; - - ret = arm_bL_ops->init_opp_table(cpu_dev); - if (ret) { - dev_err(cpu_dev, "%s: init_opp_table failed, cpu: %d, err: %d\n", - __func__, cpu_dev->id, ret); - goto atomic_dec; - } - - ret = opp_init_cpufreq_table(cpu_dev, &freq_table[cluster]); - if (ret) { - dev_err(cpu_dev, "%s: failed to init cpufreq table, cpu: %d, err: %d\n", - __func__, cpu_dev->id, ret); - goto atomic_dec; - } - - name[12] = cluster + '0'; - clk[cluster] = clk_get_sys(name, NULL); - if (!IS_ERR(clk[cluster])) { - dev_dbg(cpu_dev, "%s: clk: %p & freq table: %p, cluster: %d\n", - __func__, clk[cluster], freq_table[cluster], - cluster); - return 0; - } - - dev_err(cpu_dev, "%s: Failed to get clk for cpu: %d, cluster: %d\n", - __func__, cpu_dev->id, cluster); - ret = PTR_ERR(clk[cluster]); - opp_free_cpufreq_table(cpu_dev, &freq_table[cluster]); - -atomic_dec: - atomic_dec(&cluster_usage[cluster]); - dev_err(cpu_dev, "%s: Failed to get data for cluster: %d\n", __func__, - cluster); - return ret; -} - -/* Per-CPU initialization */ -static int bL_cpufreq_init(struct cpufreq_policy *policy) -{ - u32 cur_cluster = cpu_to_cluster(policy->cpu); - struct device *cpu_dev; - int ret; - - cpu_dev = get_cpu_device(policy->cpu); - if (!cpu_dev) { - pr_err("%s: failed to get cpu%d device\n", __func__, - policy->cpu); - return -ENODEV; - } - - ret = get_cluster_clk_and_freq_table(cpu_dev); - if (ret) - return ret; - - ret = cpufreq_frequency_table_cpuinfo(policy, freq_table[cur_cluster]); - if (ret) { - dev_err(cpu_dev, "CPU %d, cluster: %d invalid freq table\n", - policy->cpu, cur_cluster); - put_cluster_clk_and_freq_table(cpu_dev); - return ret; - } - - cpufreq_frequency_table_get_attr(freq_table[cur_cluster], policy->cpu); - - if (arm_bL_ops->get_transition_latency) - policy->cpuinfo.transition_latency = - arm_bL_ops->get_transition_latency(cpu_dev); - else - policy->cpuinfo.transition_latency = CPUFREQ_ETERNAL; - - policy->cur = bL_cpufreq_get(policy->cpu); - - cpumask_copy(policy->cpus, topology_core_cpumask(policy->cpu)); - - dev_info(cpu_dev, "%s: CPU %d initialized\n", __func__, policy->cpu); - return 0; -} - -static int bL_cpufreq_exit(struct cpufreq_policy *policy) -{ - struct device *cpu_dev; - - cpu_dev = get_cpu_device(policy->cpu); - if (!cpu_dev) { - pr_err("%s: failed to get cpu%d device\n", __func__, - policy->cpu); - return -ENODEV; - } - - put_cluster_clk_and_freq_table(cpu_dev); - dev_dbg(cpu_dev, "%s: Exited, cpu: %d\n", __func__, policy->cpu); - - return 0; -} - -/* Export freq_table to sysfs */ -static struct freq_attr *bL_cpufreq_attr[] = { - &cpufreq_freq_attr_scaling_available_freqs, - NULL, -}; - -static struct cpufreq_driver bL_cpufreq_driver = { - .name = "arm-big-little", - .flags = CPUFREQ_STICKY, - .verify = bL_cpufreq_verify_policy, - .target = bL_cpufreq_set_target, - .get = bL_cpufreq_get, - .init = bL_cpufreq_init, - .exit = bL_cpufreq_exit, - .have_governor_per_policy = true, - .attr = bL_cpufreq_attr, -}; - -int bL_cpufreq_register(struct cpufreq_arm_bL_ops *ops) -{ - int ret; - - if (arm_bL_ops) { - pr_debug("%s: Already registered: %s, exiting\n", __func__, - arm_bL_ops->name); - return -EBUSY; - } - - if (!ops || !strlen(ops->name) || !ops->init_opp_table) { - pr_err("%s: Invalid arm_bL_ops, exiting\n", __func__); - return -ENODEV; - } - - arm_bL_ops = ops; - - ret = cpufreq_register_driver(&bL_cpufreq_driver); - if (ret) { - pr_info("%s: Failed registering platform driver: %s, err: %d\n", - __func__, ops->name, ret); - arm_bL_ops = NULL; - } else { - pr_info("%s: Registered platform driver: %s\n", __func__, - ops->name); - } - - return ret; -} -EXPORT_SYMBOL_GPL(bL_cpufreq_register); - -void bL_cpufreq_unregister(struct cpufreq_arm_bL_ops *ops) -{ - if (arm_bL_ops != ops) { - pr_err("%s: Registered with: %s, can't unregister, exiting\n", - __func__, arm_bL_ops->name); - return; - } - - cpufreq_unregister_driver(&bL_cpufreq_driver); - pr_info("%s: Un-registered platform driver: %s\n", __func__, - arm_bL_ops->name); - arm_bL_ops = NULL; -} -EXPORT_SYMBOL_GPL(bL_cpufreq_unregister); diff --git a/drivers/cpufreq/arm_big_little.h b/drivers/cpufreq/arm_big_little.h deleted file mode 100644 index 79b2ce17884d..000000000000 --- a/drivers/cpufreq/arm_big_little.h +++ /dev/null @@ -1,45 +0,0 @@ -/* - * ARM big.LITTLE platform's CPUFreq header file - * - * Copyright (C) 2013 ARM Ltd. - * Sudeep KarkadaNagesha <sudeep.karkadanagesha@arm.com> - * - * Copyright (C) 2013 Linaro. - * Viresh Kumar <viresh.kumar@linaro.org> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - */ -#ifndef CPUFREQ_ARM_BIG_LITTLE_H -#define CPUFREQ_ARM_BIG_LITTLE_H - -#include <linux/cpufreq.h> -#include <linux/device.h> -#include <linux/types.h> - -struct cpufreq_arm_bL_ops { - char name[CPUFREQ_NAME_LEN]; - int (*get_transition_latency)(struct device *cpu_dev); - - /* - * This must set opp table for cpu_dev in a similar way as done by - * of_init_opp_table(). - */ - int (*init_opp_table)(struct device *cpu_dev); -}; - -static inline int cpu_to_cluster(int cpu) -{ - return topology_physical_package_id(cpu); -} - -int bL_cpufreq_register(struct cpufreq_arm_bL_ops *ops); -void bL_cpufreq_unregister(struct cpufreq_arm_bL_ops *ops); - -#endif /* CPUFREQ_ARM_BIG_LITTLE_H */ diff --git a/drivers/cpufreq/arm_big_little_dt.c b/drivers/cpufreq/arm_big_little_dt.c deleted file mode 100644 index fd9e3ea6a480..000000000000 --- a/drivers/cpufreq/arm_big_little_dt.c +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Generic big.LITTLE CPUFreq Interface driver - * - * It provides necessary ops to arm_big_little cpufreq driver and gets - * Frequency information from Device Tree. Freq table in DT must be in KHz. - * - * Copyright (C) 2013 Linaro. - * Viresh Kumar <viresh.kumar@linaro.org> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - */ - -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - -#include <linux/cpu.h> -#include <linux/cpufreq.h> -#include <linux/device.h> -#include <linux/export.h> -#include <linux/module.h> -#include <linux/of.h> -#include <linux/opp.h> -#include <linux/platform_device.h> -#include <linux/slab.h> -#include <linux/types.h> -#include "arm_big_little.h" - -/* get cpu node with valid operating-points */ -static struct device_node *get_cpu_node_with_valid_op(int cpu) -{ - struct device_node *np = NULL, *parent; - int count = 0; - - parent = of_find_node_by_path("/cpus"); - if (!parent) { - pr_err("failed to find OF /cpus\n"); - return NULL; - } - - for_each_child_of_node(parent, np) { - if (count++ != cpu) - continue; - if (!of_get_property(np, "operating-points", NULL)) { - of_node_put(np); - np = NULL; - } - - break; - } - - of_node_put(parent); - return np; -} - -static int dt_init_opp_table(struct device *cpu_dev) -{ - struct device_node *np; - int ret; - - np = get_cpu_node_with_valid_op(cpu_dev->id); - if (!np) - return -ENODATA; - - cpu_dev->of_node = np; - ret = of_init_opp_table(cpu_dev); - of_node_put(np); - - return ret; -} - -static int dt_get_transition_latency(struct device *cpu_dev) -{ - struct device_node *np; - u32 transition_latency = CPUFREQ_ETERNAL; - - np = get_cpu_node_with_valid_op(cpu_dev->id); - if (!np) - return CPUFREQ_ETERNAL; - - of_property_read_u32(np, "clock-latency", &transition_latency); - of_node_put(np); - - pr_debug("%s: clock-latency: %d\n", __func__, transition_latency); - return transition_latency; -} - -static struct cpufreq_arm_bL_ops dt_bL_ops = { - .name = "dt-bl", - .get_transition_latency = dt_get_transition_latency, - .init_opp_table = dt_init_opp_table, -}; - -static int generic_bL_probe(struct platform_device *pdev) -{ - struct device_node *np; - - np = get_cpu_node_with_valid_op(0); - if (!np) - return -ENODEV; - - of_node_put(np); - return bL_cpufreq_register(&dt_bL_ops); -} - -static int generic_bL_remove(struct platform_device *pdev) -{ - bL_cpufreq_unregister(&dt_bL_ops); - return 0; -} - -static struct platform_driver generic_bL_platdrv = { - .driver = { - .name = "arm-bL-cpufreq-dt", - .owner = THIS_MODULE, - }, - .probe = generic_bL_probe, - .remove = generic_bL_remove, -}; -module_platform_driver(generic_bL_platdrv); - -MODULE_AUTHOR("Viresh Kumar <viresh.kumar@linaro.org>"); -MODULE_DESCRIPTION("Generic ARM big LITTLE cpufreq driver via DT"); -MODULE_LICENSE("GPL"); diff --git a/drivers/cpufreq/armada-37xx-cpufreq.c b/drivers/cpufreq/armada-37xx-cpufreq.c new file mode 100644 index 000000000000..0efe403a5980 --- /dev/null +++ b/drivers/cpufreq/armada-37xx-cpufreq.c @@ -0,0 +1,558 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * CPU frequency scaling support for Armada 37xx platform. + * + * Copyright (C) 2017 Marvell + * + * Gregory CLEMENT <gregory.clement@free-electrons.com> + */ + +#include <linux/clk.h> +#include <linux/cpu.h> +#include <linux/cpufreq.h> +#include <linux/err.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/mfd/syscon.h> +#include <linux/mod_devicetable.h> +#include <linux/module.h> +#include <linux/platform_device.h> +#include <linux/pm_opp.h> +#include <linux/regmap.h> +#include <linux/slab.h> + +#include "cpufreq-dt.h" + +/* Clk register set */ +#define ARMADA_37XX_CLK_TBG_SEL 0 +#define ARMADA_37XX_CLK_TBG_SEL_CPU_OFF 22 + +/* Power management in North Bridge register set */ +#define ARMADA_37XX_NB_L0L1 0x18 +#define ARMADA_37XX_NB_L2L3 0x1C +#define ARMADA_37XX_NB_TBG_DIV_OFF 13 +#define ARMADA_37XX_NB_TBG_DIV_MASK 0x7 +#define ARMADA_37XX_NB_CLK_SEL_OFF 11 +#define ARMADA_37XX_NB_CLK_SEL_MASK 0x1 +#define ARMADA_37XX_NB_CLK_SEL_TBG 0x1 +#define ARMADA_37XX_NB_TBG_SEL_OFF 9 +#define ARMADA_37XX_NB_TBG_SEL_MASK 0x3 +#define ARMADA_37XX_NB_VDD_SEL_OFF 6 +#define ARMADA_37XX_NB_VDD_SEL_MASK 0x3 +#define ARMADA_37XX_NB_CONFIG_SHIFT 16 +#define ARMADA_37XX_NB_DYN_MOD 0x24 +#define ARMADA_37XX_NB_CLK_SEL_EN BIT(26) +#define ARMADA_37XX_NB_TBG_EN BIT(28) +#define ARMADA_37XX_NB_DIV_EN BIT(29) +#define ARMADA_37XX_NB_VDD_EN BIT(30) +#define ARMADA_37XX_NB_DFS_EN BIT(31) +#define ARMADA_37XX_NB_CPU_LOAD 0x30 +#define ARMADA_37XX_NB_CPU_LOAD_MASK 0x3 +#define ARMADA_37XX_DVFS_LOAD_0 0 +#define ARMADA_37XX_DVFS_LOAD_1 1 +#define ARMADA_37XX_DVFS_LOAD_2 2 +#define ARMADA_37XX_DVFS_LOAD_3 3 + +/* AVS register set */ +#define ARMADA_37XX_AVS_CTL0 0x0 +#define ARMADA_37XX_AVS_ENABLE BIT(30) +#define ARMADA_37XX_AVS_HIGH_VDD_LIMIT 16 +#define ARMADA_37XX_AVS_LOW_VDD_LIMIT 22 +#define ARMADA_37XX_AVS_VDD_MASK 0x3F +#define ARMADA_37XX_AVS_CTL2 0x8 +#define ARMADA_37XX_AVS_LOW_VDD_EN BIT(6) +#define ARMADA_37XX_AVS_VSET(x) (0x1C + 4 * (x)) + +/* + * On Armada 37xx the Power management manages 4 level of CPU load, + * each level can be associated with a CPU clock source, a CPU + * divider, a VDD level, etc... + */ +#define LOAD_LEVEL_NR 4 + +#define MIN_VOLT_MV 1000 +#define MIN_VOLT_MV_FOR_L1_1000MHZ 1108 +#define MIN_VOLT_MV_FOR_L1_1200MHZ 1155 + +/* AVS value for the corresponding voltage (in mV) */ +static int avs_map[] = { + 747, 758, 770, 782, 793, 805, 817, 828, 840, 852, 863, 875, 887, 898, + 910, 922, 933, 945, 957, 968, 980, 992, 1003, 1015, 1027, 1038, 1050, + 1062, 1073, 1085, 1097, 1108, 1120, 1132, 1143, 1155, 1167, 1178, 1190, + 1202, 1213, 1225, 1237, 1248, 1260, 1272, 1283, 1295, 1307, 1318, 1330, + 1342 +}; + +struct armada37xx_cpufreq_state { + struct platform_device *pdev; + struct device *cpu_dev; + struct regmap *regmap; + u32 nb_l0l1; + u32 nb_l2l3; + u32 nb_dyn_mod; + u32 nb_cpu_load; +}; + +static struct armada37xx_cpufreq_state *armada37xx_cpufreq_state; + +struct armada_37xx_dvfs { + u32 cpu_freq_max; + u8 divider[LOAD_LEVEL_NR]; + u32 avs[LOAD_LEVEL_NR]; +}; + +static struct armada_37xx_dvfs armada_37xx_dvfs[] = { + {.cpu_freq_max = 1200*1000*1000, .divider = {1, 2, 4, 6} }, + {.cpu_freq_max = 1000*1000*1000, .divider = {1, 2, 4, 5} }, + {.cpu_freq_max = 800*1000*1000, .divider = {1, 2, 3, 4} }, + {.cpu_freq_max = 600*1000*1000, .divider = {2, 4, 5, 6} }, +}; + +static struct armada_37xx_dvfs *armada_37xx_cpu_freq_info_get(u32 freq) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(armada_37xx_dvfs); i++) { + if (freq == armada_37xx_dvfs[i].cpu_freq_max) + return &armada_37xx_dvfs[i]; + } + + pr_err("Unsupported CPU frequency %d MHz\n", freq/1000000); + return NULL; +} + +/* + * Setup the four level managed by the hardware. Once the four level + * will be configured then the DVFS will be enabled. + */ +static void __init armada37xx_cpufreq_dvfs_setup(struct regmap *base, + struct regmap *clk_base, u8 *divider) +{ + u32 cpu_tbg_sel; + int load_lvl; + + /* Determine to which TBG clock is CPU connected */ + regmap_read(clk_base, ARMADA_37XX_CLK_TBG_SEL, &cpu_tbg_sel); + cpu_tbg_sel >>= ARMADA_37XX_CLK_TBG_SEL_CPU_OFF; + cpu_tbg_sel &= ARMADA_37XX_NB_TBG_SEL_MASK; + + for (load_lvl = 0; load_lvl < LOAD_LEVEL_NR; load_lvl++) { + unsigned int reg, mask, val, offset = 0; + + if (load_lvl <= ARMADA_37XX_DVFS_LOAD_1) + reg = ARMADA_37XX_NB_L0L1; + else + reg = ARMADA_37XX_NB_L2L3; + + if (load_lvl == ARMADA_37XX_DVFS_LOAD_0 || + load_lvl == ARMADA_37XX_DVFS_LOAD_2) + offset += ARMADA_37XX_NB_CONFIG_SHIFT; + + /* Set cpu clock source, for all the level we use TBG */ + val = ARMADA_37XX_NB_CLK_SEL_TBG << ARMADA_37XX_NB_CLK_SEL_OFF; + mask = (ARMADA_37XX_NB_CLK_SEL_MASK + << ARMADA_37XX_NB_CLK_SEL_OFF); + + /* Set TBG index, for all levels we use the same TBG */ + val = cpu_tbg_sel << ARMADA_37XX_NB_TBG_SEL_OFF; + mask = (ARMADA_37XX_NB_TBG_SEL_MASK + << ARMADA_37XX_NB_TBG_SEL_OFF); + + /* + * Set cpu divider based on the pre-computed array in + * order to have balanced step. + */ + val |= divider[load_lvl] << ARMADA_37XX_NB_TBG_DIV_OFF; + mask |= (ARMADA_37XX_NB_TBG_DIV_MASK + << ARMADA_37XX_NB_TBG_DIV_OFF); + + /* Set VDD divider which is actually the load level. */ + val |= load_lvl << ARMADA_37XX_NB_VDD_SEL_OFF; + mask |= (ARMADA_37XX_NB_VDD_SEL_MASK + << ARMADA_37XX_NB_VDD_SEL_OFF); + + val <<= offset; + mask <<= offset; + + regmap_update_bits(base, reg, mask, val); + } +} + +/* + * Find out the armada 37x supported AVS value whose voltage value is + * the round-up closest to the target voltage value. + */ +static u32 armada_37xx_avs_val_match(int target_vm) +{ + u32 avs; + + /* Find out the round-up closest supported voltage value */ + for (avs = 0; avs < ARRAY_SIZE(avs_map); avs++) + if (avs_map[avs] >= target_vm) + break; + + /* + * If all supported voltages are smaller than target one, + * choose the largest supported voltage + */ + if (avs == ARRAY_SIZE(avs_map)) + avs = ARRAY_SIZE(avs_map) - 1; + + return avs; +} + +/* + * For Armada 37xx soc, L0(VSET0) VDD AVS value is set to SVC revision + * value or a default value when SVC is not supported. + * - L0 can be read out from the register of AVS_CTRL_0 and L0 voltage + * can be got from the mapping table of avs_map. + * - L1 voltage should be about 100mv smaller than L0 voltage + * - L2 & L3 voltage should be about 150mv smaller than L0 voltage. + * This function calculates L1 & L2 & L3 AVS values dynamically based + * on L0 voltage and fill all AVS values to the AVS value table. + * When base CPU frequency is 1000 or 1200 MHz then there is additional + * minimal avs value for load L1. + */ +static void __init armada37xx_cpufreq_avs_configure(struct regmap *base, + struct armada_37xx_dvfs *dvfs) +{ + unsigned int target_vm; + int load_level = 0; + u32 l0_vdd_min; + + if (base == NULL) + return; + + /* Get L0 VDD min value */ + regmap_read(base, ARMADA_37XX_AVS_CTL0, &l0_vdd_min); + l0_vdd_min = (l0_vdd_min >> ARMADA_37XX_AVS_LOW_VDD_LIMIT) & + ARMADA_37XX_AVS_VDD_MASK; + if (l0_vdd_min >= ARRAY_SIZE(avs_map)) { + pr_err("L0 VDD MIN %d is not correct.\n", l0_vdd_min); + return; + } + dvfs->avs[0] = l0_vdd_min; + + if (avs_map[l0_vdd_min] <= MIN_VOLT_MV) { + /* + * If L0 voltage is smaller than 1000mv, then all VDD sets + * use L0 voltage; + */ + u32 avs_min = armada_37xx_avs_val_match(MIN_VOLT_MV); + + for (load_level = 1; load_level < LOAD_LEVEL_NR; load_level++) + dvfs->avs[load_level] = avs_min; + + /* + * Set the avs values for load L0 and L1 when base CPU frequency + * is 1000/1200 MHz to its typical initial values according to + * the Armada 3700 Hardware Specifications. + */ + if (dvfs->cpu_freq_max >= 1000*1000*1000) { + if (dvfs->cpu_freq_max >= 1200*1000*1000) + avs_min = armada_37xx_avs_val_match(MIN_VOLT_MV_FOR_L1_1200MHZ); + else + avs_min = armada_37xx_avs_val_match(MIN_VOLT_MV_FOR_L1_1000MHZ); + dvfs->avs[0] = dvfs->avs[1] = avs_min; + } + + return; + } + + /* + * L1 voltage is equal to L0 voltage - 100mv and it must be + * larger than 1000mv + */ + + target_vm = avs_map[l0_vdd_min] - 100; + target_vm = max(target_vm, MIN_VOLT_MV); + dvfs->avs[1] = armada_37xx_avs_val_match(target_vm); + + /* + * L2 & L3 voltage is equal to L0 voltage - 150mv and it must + * be larger than 1000mv + */ + target_vm = avs_map[l0_vdd_min] - 150; + target_vm = max(target_vm, MIN_VOLT_MV); + dvfs->avs[2] = dvfs->avs[3] = armada_37xx_avs_val_match(target_vm); + + /* + * Fix the avs value for load L1 when base CPU frequency is 1000/1200 MHz, + * otherwise the CPU gets stuck when switching from load L1 to load L0. + * Also ensure that avs value for load L1 is not higher than for L0. + */ + if (dvfs->cpu_freq_max >= 1000*1000*1000) { + u32 avs_min_l1; + + if (dvfs->cpu_freq_max >= 1200*1000*1000) + avs_min_l1 = armada_37xx_avs_val_match(MIN_VOLT_MV_FOR_L1_1200MHZ); + else + avs_min_l1 = armada_37xx_avs_val_match(MIN_VOLT_MV_FOR_L1_1000MHZ); + + if (avs_min_l1 > dvfs->avs[0]) + avs_min_l1 = dvfs->avs[0]; + + if (dvfs->avs[1] < avs_min_l1) + dvfs->avs[1] = avs_min_l1; + } +} + +static void __init armada37xx_cpufreq_avs_setup(struct regmap *base, + struct armada_37xx_dvfs *dvfs) +{ + unsigned int avs_val = 0; + int load_level = 0; + + if (base == NULL) + return; + + /* Disable AVS before the configuration */ + regmap_update_bits(base, ARMADA_37XX_AVS_CTL0, + ARMADA_37XX_AVS_ENABLE, 0); + + + /* Enable low voltage mode */ + regmap_update_bits(base, ARMADA_37XX_AVS_CTL2, + ARMADA_37XX_AVS_LOW_VDD_EN, + ARMADA_37XX_AVS_LOW_VDD_EN); + + + for (load_level = 1; load_level < LOAD_LEVEL_NR; load_level++) { + avs_val = dvfs->avs[load_level]; + regmap_update_bits(base, ARMADA_37XX_AVS_VSET(load_level-1), + ARMADA_37XX_AVS_VDD_MASK << ARMADA_37XX_AVS_HIGH_VDD_LIMIT | + ARMADA_37XX_AVS_VDD_MASK << ARMADA_37XX_AVS_LOW_VDD_LIMIT, + avs_val << ARMADA_37XX_AVS_HIGH_VDD_LIMIT | + avs_val << ARMADA_37XX_AVS_LOW_VDD_LIMIT); + } + + /* Enable AVS after the configuration */ + regmap_update_bits(base, ARMADA_37XX_AVS_CTL0, + ARMADA_37XX_AVS_ENABLE, + ARMADA_37XX_AVS_ENABLE); + +} + +static void armada37xx_cpufreq_disable_dvfs(struct regmap *base) +{ + unsigned int reg = ARMADA_37XX_NB_DYN_MOD, + mask = ARMADA_37XX_NB_DFS_EN; + + regmap_update_bits(base, reg, mask, 0); +} + +static void __init armada37xx_cpufreq_enable_dvfs(struct regmap *base) +{ + unsigned int val, reg = ARMADA_37XX_NB_CPU_LOAD, + mask = ARMADA_37XX_NB_CPU_LOAD_MASK; + + /* Start with the highest load (0) */ + val = ARMADA_37XX_DVFS_LOAD_0; + regmap_update_bits(base, reg, mask, val); + + /* Now enable DVFS for the CPUs */ + reg = ARMADA_37XX_NB_DYN_MOD; + mask = ARMADA_37XX_NB_CLK_SEL_EN | ARMADA_37XX_NB_TBG_EN | + ARMADA_37XX_NB_DIV_EN | ARMADA_37XX_NB_VDD_EN | + ARMADA_37XX_NB_DFS_EN; + + regmap_update_bits(base, reg, mask, mask); +} + +static int armada37xx_cpufreq_suspend(struct cpufreq_policy *policy) +{ + struct armada37xx_cpufreq_state *state = armada37xx_cpufreq_state; + + regmap_read(state->regmap, ARMADA_37XX_NB_L0L1, &state->nb_l0l1); + regmap_read(state->regmap, ARMADA_37XX_NB_L2L3, &state->nb_l2l3); + regmap_read(state->regmap, ARMADA_37XX_NB_CPU_LOAD, + &state->nb_cpu_load); + regmap_read(state->regmap, ARMADA_37XX_NB_DYN_MOD, &state->nb_dyn_mod); + + return 0; +} + +static int armada37xx_cpufreq_resume(struct cpufreq_policy *policy) +{ + struct armada37xx_cpufreq_state *state = armada37xx_cpufreq_state; + + /* Ensure DVFS is disabled otherwise the following registers are RO */ + armada37xx_cpufreq_disable_dvfs(state->regmap); + + regmap_write(state->regmap, ARMADA_37XX_NB_L0L1, state->nb_l0l1); + regmap_write(state->regmap, ARMADA_37XX_NB_L2L3, state->nb_l2l3); + regmap_write(state->regmap, ARMADA_37XX_NB_CPU_LOAD, + state->nb_cpu_load); + + /* + * NB_DYN_MOD register is the one that actually enable back DVFS if it + * was enabled before the suspend operation. This must be done last + * otherwise other registers are not writable. + */ + regmap_write(state->regmap, ARMADA_37XX_NB_DYN_MOD, state->nb_dyn_mod); + + return 0; +} + +static int __init armada37xx_cpufreq_driver_init(void) +{ + struct cpufreq_dt_platform_data pdata; + struct armada_37xx_dvfs *dvfs; + struct platform_device *pdev; + unsigned long freq; + unsigned int base_frequency; + struct regmap *nb_clk_base, *nb_pm_base, *avs_base; + struct device *cpu_dev; + int load_lvl, ret; + struct clk *clk, *parent; + + nb_clk_base = + syscon_regmap_lookup_by_compatible("marvell,armada-3700-periph-clock-nb"); + if (IS_ERR(nb_clk_base)) + return -ENODEV; + + nb_pm_base = + syscon_regmap_lookup_by_compatible("marvell,armada-3700-nb-pm"); + + if (IS_ERR(nb_pm_base)) + return -ENODEV; + + avs_base = + syscon_regmap_lookup_by_compatible("marvell,armada-3700-avs"); + + /* if AVS is not present don't use it but still try to setup dvfs */ + if (IS_ERR(avs_base)) { + pr_info("Syscon failed for Adapting Voltage Scaling: skip it\n"); + avs_base = NULL; + } + /* Before doing any configuration on the DVFS first, disable it */ + armada37xx_cpufreq_disable_dvfs(nb_pm_base); + + /* + * On CPU 0 register the operating points supported (which are + * the nominal CPU frequency and full integer divisions of + * it). + */ + cpu_dev = get_cpu_device(0); + if (!cpu_dev) { + dev_err(cpu_dev, "Cannot get CPU\n"); + return -ENODEV; + } + + clk = clk_get(cpu_dev, NULL); + if (IS_ERR(clk)) { + dev_err(cpu_dev, "Cannot get clock for CPU0\n"); + return PTR_ERR(clk); + } + + parent = clk_get_parent(clk); + if (IS_ERR(parent)) { + dev_err(cpu_dev, "Cannot get parent clock for CPU0\n"); + clk_put(clk); + return PTR_ERR(parent); + } + + /* Get parent CPU frequency */ + base_frequency = clk_get_rate(parent); + + if (!base_frequency) { + dev_err(cpu_dev, "Failed to get parent clock rate for CPU\n"); + clk_put(clk); + return -EINVAL; + } + + dvfs = armada_37xx_cpu_freq_info_get(base_frequency); + if (!dvfs) { + clk_put(clk); + return -EINVAL; + } + + armada37xx_cpufreq_state = kmalloc(sizeof(*armada37xx_cpufreq_state), + GFP_KERNEL); + if (!armada37xx_cpufreq_state) { + clk_put(clk); + return -ENOMEM; + } + + armada37xx_cpufreq_state->regmap = nb_pm_base; + + armada37xx_cpufreq_avs_configure(avs_base, dvfs); + armada37xx_cpufreq_avs_setup(avs_base, dvfs); + + armada37xx_cpufreq_dvfs_setup(nb_pm_base, nb_clk_base, dvfs->divider); + clk_put(clk); + + for (load_lvl = ARMADA_37XX_DVFS_LOAD_0; load_lvl < LOAD_LEVEL_NR; + load_lvl++) { + unsigned long u_volt = avs_map[dvfs->avs[load_lvl]] * 1000; + freq = base_frequency / dvfs->divider[load_lvl]; + ret = dev_pm_opp_add(cpu_dev, freq, u_volt); + if (ret) + goto remove_opp; + + + } + + /* Now that everything is setup, enable the DVFS at hardware level */ + armada37xx_cpufreq_enable_dvfs(nb_pm_base); + + memset(&pdata, 0, sizeof(pdata)); + pdata.suspend = armada37xx_cpufreq_suspend; + pdata.resume = armada37xx_cpufreq_resume; + + pdev = platform_device_register_data(NULL, "cpufreq-dt", -1, &pdata, + sizeof(pdata)); + ret = PTR_ERR_OR_ZERO(pdev); + if (ret) + goto disable_dvfs; + + armada37xx_cpufreq_state->cpu_dev = cpu_dev; + armada37xx_cpufreq_state->pdev = pdev; + platform_set_drvdata(pdev, dvfs); + return 0; + +disable_dvfs: + armada37xx_cpufreq_disable_dvfs(nb_pm_base); +remove_opp: + /* clean-up the already added opp before leaving */ + while (load_lvl-- > ARMADA_37XX_DVFS_LOAD_0) { + freq = base_frequency / dvfs->divider[load_lvl]; + dev_pm_opp_remove(cpu_dev, freq); + } + + kfree(armada37xx_cpufreq_state); + + return ret; +} +/* late_initcall, to guarantee the driver is loaded after A37xx clock driver */ +late_initcall(armada37xx_cpufreq_driver_init); + +static void __exit armada37xx_cpufreq_driver_exit(void) +{ + struct platform_device *pdev = armada37xx_cpufreq_state->pdev; + struct armada_37xx_dvfs *dvfs = platform_get_drvdata(pdev); + unsigned long freq; + int load_lvl; + + platform_device_unregister(pdev); + + armada37xx_cpufreq_disable_dvfs(armada37xx_cpufreq_state->regmap); + + for (load_lvl = ARMADA_37XX_DVFS_LOAD_0; load_lvl < LOAD_LEVEL_NR; load_lvl++) { + freq = dvfs->cpu_freq_max / dvfs->divider[load_lvl]; + dev_pm_opp_remove(armada37xx_cpufreq_state->cpu_dev, freq); + } + + kfree(armada37xx_cpufreq_state); +} +module_exit(armada37xx_cpufreq_driver_exit); + +static const struct of_device_id __maybe_unused armada37xx_cpufreq_of_match[] = { + { .compatible = "marvell,armada-3700-nb-pm" }, + { }, +}; +MODULE_DEVICE_TABLE(of, armada37xx_cpufreq_of_match); + +MODULE_AUTHOR("Gregory CLEMENT <gregory.clement@free-electrons.com>"); +MODULE_DESCRIPTION("Armada 37xx cpufreq driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/cpufreq/armada-8k-cpufreq.c b/drivers/cpufreq/armada-8k-cpufreq.c new file mode 100644 index 000000000000..d96c1718f7f8 --- /dev/null +++ b/drivers/cpufreq/armada-8k-cpufreq.c @@ -0,0 +1,216 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * CPUFreq support for Armada 8K + * + * Copyright (C) 2018 Marvell + * + * Omri Itach <omrii@marvell.com> + * Gregory Clement <gregory.clement@bootlin.com> + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/clk.h> +#include <linux/cpu.h> +#include <linux/err.h> +#include <linux/init.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/platform_device.h> +#include <linux/pm_opp.h> +#include <linux/slab.h> + +static const struct of_device_id __maybe_unused armada_8k_cpufreq_of_match[] = { + { .compatible = "marvell,ap806-cpu-clock" }, + { .compatible = "marvell,ap807-cpu-clock" }, + { }, +}; +MODULE_DEVICE_TABLE(of, armada_8k_cpufreq_of_match); + +/* + * Setup the opps list with the divider for the max frequency, that + * will be filled at runtime. + */ +static const int opps_div[] __initconst = {1, 2, 3, 4}; + +static struct platform_device *armada_8k_pdev; + +struct freq_table { + struct device *cpu_dev; + unsigned int freq[ARRAY_SIZE(opps_div)]; +}; + +/* If the CPUs share the same clock, then they are in the same cluster. */ +static void __init armada_8k_get_sharing_cpus(struct clk *cur_clk, + struct cpumask *cpumask) +{ + int cpu; + + for_each_present_cpu(cpu) { + struct device *cpu_dev; + struct clk *clk; + + cpu_dev = get_cpu_device(cpu); + if (!cpu_dev) { + pr_warn("Failed to get cpu%d device\n", cpu); + continue; + } + + clk = clk_get(cpu_dev, NULL); + if (IS_ERR(clk)) { + pr_warn("Cannot get clock for CPU %d\n", cpu); + } else { + if (clk_is_match(clk, cur_clk)) + cpumask_set_cpu(cpu, cpumask); + + clk_put(clk); + } + } +} + +static int __init armada_8k_add_opp(struct clk *clk, struct device *cpu_dev, + struct freq_table *freq_tables, + int opps_index) +{ + unsigned int cur_frequency; + unsigned int freq; + int i, ret; + + /* Get nominal (current) CPU frequency. */ + cur_frequency = clk_get_rate(clk); + if (!cur_frequency) { + dev_err(cpu_dev, "Failed to get clock rate for this CPU\n"); + return -EINVAL; + } + + freq_tables[opps_index].cpu_dev = cpu_dev; + + for (i = 0; i < ARRAY_SIZE(opps_div); i++) { + freq = cur_frequency / opps_div[i]; + + ret = dev_pm_opp_add(cpu_dev, freq, 0); + if (ret) + return ret; + + freq_tables[opps_index].freq[i] = freq; + } + + return 0; +} + +static void armada_8k_cpufreq_free_table(struct freq_table *freq_tables) +{ + int opps_index, nb_cpus = num_possible_cpus(); + + for (opps_index = 0 ; opps_index < nb_cpus; opps_index++) { + int i; + + /* If cpu_dev is NULL then we reached the end of the array */ + if (!freq_tables[opps_index].cpu_dev) + break; + + for (i = 0; i < ARRAY_SIZE(opps_div); i++) { + /* + * A 0Hz frequency is not valid, this meant + * that it was not yet initialized so there is + * no more opp to free + */ + if (freq_tables[opps_index].freq[i] == 0) + break; + + dev_pm_opp_remove(freq_tables[opps_index].cpu_dev, + freq_tables[opps_index].freq[i]); + } + } + + kfree(freq_tables); +} + +static int __init armada_8k_cpufreq_init(void) +{ + int ret = 0, opps_index = 0, cpu, nb_cpus; + struct freq_table *freq_tables; + struct device_node *node; + static struct cpumask cpus, shared_cpus; + + node = of_find_matching_node_and_match(NULL, armada_8k_cpufreq_of_match, + NULL); + if (!node || !of_device_is_available(node)) { + of_node_put(node); + return -ENODEV; + } + of_node_put(node); + + nb_cpus = num_possible_cpus(); + freq_tables = kcalloc(nb_cpus, sizeof(*freq_tables), GFP_KERNEL); + if (!freq_tables) + return -ENOMEM; + cpumask_copy(&cpus, cpu_possible_mask); + + /* + * For each CPU, this loop registers the operating points + * supported (which are the nominal CPU frequency and full integer + * divisions of it). + */ + for_each_cpu(cpu, &cpus) { + struct device *cpu_dev; + struct clk *clk; + + cpu_dev = get_cpu_device(cpu); + + if (!cpu_dev) { + pr_err("Cannot get CPU %d\n", cpu); + continue; + } + + clk = clk_get(cpu_dev, NULL); + + if (IS_ERR(clk)) { + pr_err("Cannot get clock for CPU %d\n", cpu); + ret = PTR_ERR(clk); + goto remove_opp; + } + + ret = armada_8k_add_opp(clk, cpu_dev, freq_tables, opps_index); + if (ret) { + clk_put(clk); + goto remove_opp; + } + + opps_index++; + cpumask_clear(&shared_cpus); + armada_8k_get_sharing_cpus(clk, &shared_cpus); + dev_pm_opp_set_sharing_cpus(cpu_dev, &shared_cpus); + cpumask_andnot(&cpus, &cpus, &shared_cpus); + clk_put(clk); + } + + armada_8k_pdev = platform_device_register_simple("cpufreq-dt", -1, + NULL, 0); + ret = PTR_ERR_OR_ZERO(armada_8k_pdev); + if (ret) + goto remove_opp; + + platform_set_drvdata(armada_8k_pdev, freq_tables); + + return 0; + +remove_opp: + armada_8k_cpufreq_free_table(freq_tables); + return ret; +} +module_init(armada_8k_cpufreq_init); + +static void __exit armada_8k_cpufreq_exit(void) +{ + struct freq_table *freq_tables = platform_get_drvdata(armada_8k_pdev); + + platform_device_unregister(armada_8k_pdev); + armada_8k_cpufreq_free_table(freq_tables); +} +module_exit(armada_8k_cpufreq_exit); + +MODULE_AUTHOR("Gregory Clement <gregory.clement@bootlin.com>"); +MODULE_DESCRIPTION("Armada 8K cpufreq driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/cpufreq/at32ap-cpufreq.c b/drivers/cpufreq/at32ap-cpufreq.c deleted file mode 100644 index 654488723cb5..000000000000 --- a/drivers/cpufreq/at32ap-cpufreq.c +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Copyright (C) 2004-2007 Atmel Corporation - * - * Based on MIPS implementation arch/mips/kernel/time.c - * Copyright 2001 MontaVista Software Inc. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - */ - -/*#define DEBUG*/ - -#include <linux/kernel.h> -#include <linux/types.h> -#include <linux/init.h> -#include <linux/cpufreq.h> -#include <linux/io.h> -#include <linux/clk.h> -#include <linux/err.h> -#include <linux/export.h> - -static struct clk *cpuclk; - -static int at32_verify_speed(struct cpufreq_policy *policy) -{ - if (policy->cpu != 0) - return -EINVAL; - - cpufreq_verify_within_limits(policy, policy->cpuinfo.min_freq, - policy->cpuinfo.max_freq); - return 0; -} - -static unsigned int at32_get_speed(unsigned int cpu) -{ - /* No SMP support */ - if (cpu) - return 0; - return (unsigned int)((clk_get_rate(cpuclk) + 500) / 1000); -} - -static unsigned int ref_freq; -static unsigned long loops_per_jiffy_ref; - -static int at32_set_target(struct cpufreq_policy *policy, - unsigned int target_freq, - unsigned int relation) -{ - struct cpufreq_freqs freqs; - long freq; - - /* Convert target_freq from kHz to Hz */ - freq = clk_round_rate(cpuclk, target_freq * 1000); - - /* Check if policy->min <= new_freq <= policy->max */ - if(freq < (policy->min * 1000) || freq > (policy->max * 1000)) - return -EINVAL; - - pr_debug("cpufreq: requested frequency %u Hz\n", target_freq * 1000); - - freqs.old = at32_get_speed(0); - freqs.new = (freq + 500) / 1000; - freqs.flags = 0; - - if (!ref_freq) { - ref_freq = freqs.old; - loops_per_jiffy_ref = boot_cpu_data.loops_per_jiffy; - } - - cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE); - if (freqs.old < freqs.new) - boot_cpu_data.loops_per_jiffy = cpufreq_scale( - loops_per_jiffy_ref, ref_freq, freqs.new); - clk_set_rate(cpuclk, freq); - if (freqs.new < freqs.old) - boot_cpu_data.loops_per_jiffy = cpufreq_scale( - loops_per_jiffy_ref, ref_freq, freqs.new); - cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); - - pr_debug("cpufreq: set frequency %lu Hz\n", freq); - - return 0; -} - -static int __init at32_cpufreq_driver_init(struct cpufreq_policy *policy) -{ - if (policy->cpu != 0) - return -EINVAL; - - cpuclk = clk_get(NULL, "cpu"); - if (IS_ERR(cpuclk)) { - pr_debug("cpufreq: could not get CPU clk\n"); - return PTR_ERR(cpuclk); - } - - policy->cpuinfo.min_freq = (clk_round_rate(cpuclk, 1) + 500) / 1000; - policy->cpuinfo.max_freq = (clk_round_rate(cpuclk, ~0UL) + 500) / 1000; - policy->cpuinfo.transition_latency = 0; - policy->cur = at32_get_speed(0); - policy->min = policy->cpuinfo.min_freq; - policy->max = policy->cpuinfo.max_freq; - - printk("cpufreq: AT32AP CPU frequency driver\n"); - - return 0; -} - -static struct cpufreq_driver at32_driver = { - .name = "at32ap", - .owner = THIS_MODULE, - .init = at32_cpufreq_driver_init, - .verify = at32_verify_speed, - .target = at32_set_target, - .get = at32_get_speed, - .flags = CPUFREQ_STICKY, -}; - -static int __init at32_cpufreq_init(void) -{ - return cpufreq_register_driver(&at32_driver); -} -late_initcall(at32_cpufreq_init); diff --git a/drivers/cpufreq/blackfin-cpufreq.c b/drivers/cpufreq/blackfin-cpufreq.c deleted file mode 100644 index 9cdbbd278a80..000000000000 --- a/drivers/cpufreq/blackfin-cpufreq.c +++ /dev/null @@ -1,247 +0,0 @@ -/* - * Blackfin core clock scaling - * - * Copyright 2008-2011 Analog Devices Inc. - * - * Licensed under the GPL-2 or later. - */ - -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/types.h> -#include <linux/init.h> -#include <linux/clk.h> -#include <linux/cpufreq.h> -#include <linux/fs.h> -#include <linux/delay.h> -#include <asm/blackfin.h> -#include <asm/time.h> -#include <asm/dpmc.h> - - -/* this is the table of CCLK frequencies, in Hz */ -/* .driver_data is the entry in the auxiliary dpm_state_table[] */ -static struct cpufreq_frequency_table bfin_freq_table[] = { - { - .frequency = CPUFREQ_TABLE_END, - .driver_data = 0, - }, - { - .frequency = CPUFREQ_TABLE_END, - .driver_data = 1, - }, - { - .frequency = CPUFREQ_TABLE_END, - .driver_data = 2, - }, - { - .frequency = CPUFREQ_TABLE_END, - .driver_data = 0, - }, -}; - -static struct bfin_dpm_state { - unsigned int csel; /* system clock divider */ - unsigned int tscale; /* change the divider on the core timer interrupt */ -} dpm_state_table[3]; - -#if defined(CONFIG_CYCLES_CLOCKSOURCE) -/* - * normalized to maximum frequency offset for CYCLES, - * used in time-ts cycles clock source, but could be used - * somewhere also. - */ -unsigned long long __bfin_cycles_off; -unsigned int __bfin_cycles_mod; -#endif - -/**************************************************************************/ -static void __init bfin_init_tables(unsigned long cclk, unsigned long sclk) -{ - - unsigned long csel, min_cclk; - int index; - - /* Anomaly 273 seems to still exist on non-BF54x w/dcache turned on */ -#if ANOMALY_05000273 || ANOMALY_05000274 || \ - (!(defined(CONFIG_BF54x) || defined(CONFIG_BF60x)) \ - && defined(CONFIG_BFIN_EXTMEM_DCACHEABLE)) - min_cclk = sclk * 2; -#else - min_cclk = sclk; -#endif - -#ifndef CONFIG_BF60x - csel = ((bfin_read_PLL_DIV() & CSEL) >> 4); -#else - csel = bfin_read32(CGU0_DIV) & 0x1F; -#endif - - for (index = 0; (cclk >> index) >= min_cclk && csel <= 3 && index < 3; index++, csel++) { - bfin_freq_table[index].frequency = cclk >> index; -#ifndef CONFIG_BF60x - dpm_state_table[index].csel = csel << 4; /* Shift now into PLL_DIV bitpos */ -#else - dpm_state_table[index].csel = csel; -#endif - dpm_state_table[index].tscale = (TIME_SCALE >> index) - 1; - - pr_debug("cpufreq: freq:%d csel:0x%x tscale:%d\n", - bfin_freq_table[index].frequency, - dpm_state_table[index].csel, - dpm_state_table[index].tscale); - } - return; -} - -static void bfin_adjust_core_timer(void *info) -{ - unsigned int tscale; - unsigned int index = *(unsigned int *)info; - - /* we have to adjust the core timer, because it is using cclk */ - tscale = dpm_state_table[index].tscale; - bfin_write_TSCALE(tscale); - return; -} - -static unsigned int bfin_getfreq_khz(unsigned int cpu) -{ - /* Both CoreA/B have the same core clock */ - return get_cclk() / 1000; -} - -#ifdef CONFIG_BF60x -unsigned long cpu_set_cclk(int cpu, unsigned long new) -{ - struct clk *clk; - int ret; - - clk = clk_get(NULL, "CCLK"); - if (IS_ERR(clk)) - return -ENODEV; - - ret = clk_set_rate(clk, new); - clk_put(clk); - return ret; -} -#endif - -static int bfin_target(struct cpufreq_policy *policy, - unsigned int target_freq, unsigned int relation) -{ -#ifndef CONFIG_BF60x - unsigned int plldiv; -#endif - unsigned int index; - unsigned long cclk_hz; - struct cpufreq_freqs freqs; - static unsigned long lpj_ref; - static unsigned int lpj_ref_freq; - int ret = 0; - -#if defined(CONFIG_CYCLES_CLOCKSOURCE) - cycles_t cycles; -#endif - - if (cpufreq_frequency_table_target(policy, bfin_freq_table, target_freq, - relation, &index)) - return -EINVAL; - - cclk_hz = bfin_freq_table[index].frequency; - - freqs.old = bfin_getfreq_khz(0); - freqs.new = cclk_hz; - - pr_debug("cpufreq: changing cclk to %lu; target = %u, oldfreq = %u\n", - cclk_hz, target_freq, freqs.old); - - cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE); -#ifndef CONFIG_BF60x - plldiv = (bfin_read_PLL_DIV() & SSEL) | dpm_state_table[index].csel; - bfin_write_PLL_DIV(plldiv); -#else - ret = cpu_set_cclk(policy->cpu, freqs.new * 1000); - if (ret != 0) { - WARN_ONCE(ret, "cpufreq set freq failed %d\n", ret); - return ret; - } -#endif - on_each_cpu(bfin_adjust_core_timer, &index, 1); -#if defined(CONFIG_CYCLES_CLOCKSOURCE) - cycles = get_cycles(); - SSYNC(); - cycles += 10; /* ~10 cycles we lose after get_cycles() */ - __bfin_cycles_off += (cycles << __bfin_cycles_mod) - (cycles << index); - __bfin_cycles_mod = index; -#endif - if (!lpj_ref_freq) { - lpj_ref = loops_per_jiffy; - lpj_ref_freq = freqs.old; - } - if (freqs.new != freqs.old) { - loops_per_jiffy = cpufreq_scale(lpj_ref, - lpj_ref_freq, freqs.new); - } - - /* TODO: just test case for cycles clock source, remove later */ - cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); - - pr_debug("cpufreq: done\n"); - return ret; -} - -static int bfin_verify_speed(struct cpufreq_policy *policy) -{ - return cpufreq_frequency_table_verify(policy, bfin_freq_table); -} - -static int __bfin_cpu_init(struct cpufreq_policy *policy) -{ - - unsigned long cclk, sclk; - - cclk = get_cclk() / 1000; - sclk = get_sclk() / 1000; - - if (policy->cpu == CPUFREQ_CPU) - bfin_init_tables(cclk, sclk); - - policy->cpuinfo.transition_latency = 50000; /* 50us assumed */ - - policy->cur = cclk; - cpufreq_frequency_table_get_attr(bfin_freq_table, policy->cpu); - return cpufreq_frequency_table_cpuinfo(policy, bfin_freq_table); -} - -static struct freq_attr *bfin_freq_attr[] = { - &cpufreq_freq_attr_scaling_available_freqs, - NULL, -}; - -static struct cpufreq_driver bfin_driver = { - .verify = bfin_verify_speed, - .target = bfin_target, - .get = bfin_getfreq_khz, - .init = __bfin_cpu_init, - .name = "bfin cpufreq", - .owner = THIS_MODULE, - .attr = bfin_freq_attr, -}; - -static int __init bfin_cpu_init(void) -{ - return cpufreq_register_driver(&bfin_driver); -} - -static void __exit bfin_cpu_exit(void) -{ - cpufreq_unregister_driver(&bfin_driver); -} - -MODULE_AUTHOR("Michael Hennerich <hennerich@blackfin.uclinux.org>"); -MODULE_DESCRIPTION("cpufreq driver for Blackfin"); -MODULE_LICENSE("GPL"); - -module_init(bfin_cpu_init); -module_exit(bfin_cpu_exit); diff --git a/drivers/cpufreq/bmips-cpufreq.c b/drivers/cpufreq/bmips-cpufreq.c new file mode 100644 index 000000000000..36051880640b --- /dev/null +++ b/drivers/cpufreq/bmips-cpufreq.c @@ -0,0 +1,186 @@ +/* + * CPU frequency scaling for Broadcom BMIPS SoCs + * + * Copyright (c) 2017 Broadcom + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation version 2. + * + * This program is distributed "as is" WITHOUT ANY WARRANTY of any + * kind, whether express or implied; without even the implied warranty + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include <linux/cpufreq.h> +#include <linux/module.h> +#include <linux/of_address.h> +#include <linux/slab.h> + +/* for mips_hpt_frequency */ +#include <asm/time.h> + +#define BMIPS_CPUFREQ_PREFIX "bmips" +#define BMIPS_CPUFREQ_NAME BMIPS_CPUFREQ_PREFIX "-cpufreq" + +#define TRANSITION_LATENCY (25 * 1000) /* 25 us */ + +#define BMIPS5_CLK_DIV_SET_SHIFT 0x7 +#define BMIPS5_CLK_DIV_SHIFT 0x4 +#define BMIPS5_CLK_DIV_MASK 0xf + +enum bmips_type { + BMIPS5000, + BMIPS5200, +}; + +struct cpufreq_compat { + const char *compatible; + unsigned int bmips_type; + unsigned int clk_mult; + unsigned int max_freqs; +}; + +#define BMIPS(c, t, m, f) { \ + .compatible = c, \ + .bmips_type = (t), \ + .clk_mult = (m), \ + .max_freqs = (f), \ +} + +static struct cpufreq_compat bmips_cpufreq_compat[] = { + BMIPS("brcm,bmips5000", BMIPS5000, 8, 4), + BMIPS("brcm,bmips5200", BMIPS5200, 8, 4), + { } +}; + +static struct cpufreq_compat *priv; + +static int htp_freq_to_cpu_freq(unsigned int clk_mult) +{ + return mips_hpt_frequency * clk_mult / 1000; +} + +static struct cpufreq_frequency_table * +bmips_cpufreq_get_freq_table(const struct cpufreq_policy *policy) +{ + struct cpufreq_frequency_table *table; + unsigned long cpu_freq; + int i; + + cpu_freq = htp_freq_to_cpu_freq(priv->clk_mult); + + table = kmalloc_array(priv->max_freqs + 1, sizeof(*table), GFP_KERNEL); + if (!table) + return ERR_PTR(-ENOMEM); + + for (i = 0; i < priv->max_freqs; i++) { + table[i].frequency = cpu_freq / (1 << i); + table[i].driver_data = i; + } + table[i].frequency = CPUFREQ_TABLE_END; + + return table; +} + +static unsigned int bmips_cpufreq_get(unsigned int cpu) +{ + unsigned int div; + uint32_t mode; + + switch (priv->bmips_type) { + case BMIPS5200: + case BMIPS5000: + mode = read_c0_brcm_mode(); + div = ((mode >> BMIPS5_CLK_DIV_SHIFT) & BMIPS5_CLK_DIV_MASK); + break; + default: + div = 0; + } + + return htp_freq_to_cpu_freq(priv->clk_mult) / (1 << div); +} + +static int bmips_cpufreq_target_index(struct cpufreq_policy *policy, + unsigned int index) +{ + unsigned int div = policy->freq_table[index].driver_data; + + switch (priv->bmips_type) { + case BMIPS5200: + case BMIPS5000: + change_c0_brcm_mode(BMIPS5_CLK_DIV_MASK << BMIPS5_CLK_DIV_SHIFT, + (1 << BMIPS5_CLK_DIV_SET_SHIFT) | + (div << BMIPS5_CLK_DIV_SHIFT)); + break; + default: + return -ENOTSUPP; + } + + return 0; +} + +static void bmips_cpufreq_exit(struct cpufreq_policy *policy) +{ + kfree(policy->freq_table); +} + +static int bmips_cpufreq_init(struct cpufreq_policy *policy) +{ + struct cpufreq_frequency_table *freq_table; + + freq_table = bmips_cpufreq_get_freq_table(policy); + if (IS_ERR(freq_table)) { + pr_err("%s: couldn't determine frequency table (%ld).\n", + BMIPS_CPUFREQ_NAME, PTR_ERR(freq_table)); + return PTR_ERR(freq_table); + } + + cpufreq_generic_init(policy, freq_table, TRANSITION_LATENCY); + pr_info("%s: registered\n", BMIPS_CPUFREQ_NAME); + + return 0; +} + +static struct cpufreq_driver bmips_cpufreq_driver = { + .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK, + .verify = cpufreq_generic_frequency_table_verify, + .target_index = bmips_cpufreq_target_index, + .get = bmips_cpufreq_get, + .init = bmips_cpufreq_init, + .exit = bmips_cpufreq_exit, + .name = BMIPS_CPUFREQ_PREFIX, +}; + +static int __init bmips_cpufreq_driver_init(void) +{ + struct cpufreq_compat *cc; + struct device_node *np; + + for (cc = bmips_cpufreq_compat; cc->compatible; cc++) { + np = of_find_compatible_node(NULL, "cpu", cc->compatible); + if (np) { + of_node_put(np); + priv = cc; + break; + } + } + + /* We hit the guard element of the array. No compatible CPU found. */ + if (!cc->compatible) + return -ENODEV; + + return cpufreq_register_driver(&bmips_cpufreq_driver); +} +module_init(bmips_cpufreq_driver_init); + +static void __exit bmips_cpufreq_driver_exit(void) +{ + cpufreq_unregister_driver(&bmips_cpufreq_driver); +} +module_exit(bmips_cpufreq_driver_exit); + +MODULE_AUTHOR("Markus Mayer <mmayer@broadcom.com>"); +MODULE_DESCRIPTION("CPUfreq driver for Broadcom BMIPS SoCs"); +MODULE_LICENSE("GPL"); diff --git a/drivers/cpufreq/brcmstb-avs-cpufreq.c b/drivers/cpufreq/brcmstb-avs-cpufreq.c new file mode 100644 index 000000000000..71450cca8e9f --- /dev/null +++ b/drivers/cpufreq/brcmstb-avs-cpufreq.c @@ -0,0 +1,783 @@ +/* + * CPU frequency scaling for Broadcom SoCs with AVS firmware that + * supports DVS or DVFS + * + * Copyright (c) 2016 Broadcom + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation version 2. + * + * This program is distributed "as is" WITHOUT ANY WARRANTY of any + * kind, whether express or implied; without even the implied warranty + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +/* + * "AVS" is the name of a firmware developed at Broadcom. It derives + * its name from the technique called "Adaptive Voltage Scaling". + * Adaptive voltage scaling was the original purpose of this firmware. + * The AVS firmware still supports "AVS mode", where all it does is + * adaptive voltage scaling. However, on some newer Broadcom SoCs, the + * AVS Firmware, despite its unchanged name, also supports DFS mode and + * DVFS mode. + * + * In the context of this document and the related driver, "AVS" by + * itself always means the Broadcom firmware and never refers to the + * technique called "Adaptive Voltage Scaling". + * + * The Broadcom STB AVS CPUfreq driver provides voltage and frequency + * scaling on Broadcom SoCs using AVS firmware with support for DFS and + * DVFS. The AVS firmware is running on its own co-processor. The + * driver supports both uniprocessor (UP) and symmetric multiprocessor + * (SMP) systems which share clock and voltage across all CPUs. + * + * Actual voltage and frequency scaling is done solely by the AVS + * firmware. This driver does not change frequency or voltage itself. + * It provides a standard CPUfreq interface to the rest of the kernel + * and to userland. It interfaces with the AVS firmware to effect the + * requested changes and to report back the current system status in a + * way that is expected by existing tools. + */ + +#include <linux/cpufreq.h> +#include <linux/delay.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/module.h> +#include <linux/of_address.h> +#include <linux/platform_device.h> +#include <linux/semaphore.h> + +/* Max number of arguments AVS calls take */ +#define AVS_MAX_CMD_ARGS 4 +/* + * This macro is used to generate AVS parameter register offsets. For + * x >= AVS_MAX_CMD_ARGS, it returns 0 to protect against accidental memory + * access outside of the parameter range. (Offset 0 is the first parameter.) + */ +#define AVS_PARAM_MULT(x) ((x) < AVS_MAX_CMD_ARGS ? (x) : 0) + +/* AVS Mailbox Register offsets */ +#define AVS_MBOX_COMMAND 0x00 +#define AVS_MBOX_STATUS 0x04 +#define AVS_MBOX_VOLTAGE0 0x08 +#define AVS_MBOX_TEMP0 0x0c +#define AVS_MBOX_PV0 0x10 +#define AVS_MBOX_MV0 0x14 +#define AVS_MBOX_PARAM(x) (0x18 + AVS_PARAM_MULT(x) * sizeof(u32)) +#define AVS_MBOX_REVISION 0x28 +#define AVS_MBOX_PSTATE 0x2c +#define AVS_MBOX_HEARTBEAT 0x30 +#define AVS_MBOX_MAGIC 0x34 +#define AVS_MBOX_SIGMA_HVT 0x38 +#define AVS_MBOX_SIGMA_SVT 0x3c +#define AVS_MBOX_VOLTAGE1 0x40 +#define AVS_MBOX_TEMP1 0x44 +#define AVS_MBOX_PV1 0x48 +#define AVS_MBOX_MV1 0x4c +#define AVS_MBOX_FREQUENCY 0x50 + +/* AVS Commands */ +#define AVS_CMD_AVAILABLE 0x00 +#define AVS_CMD_DISABLE 0x10 +#define AVS_CMD_ENABLE 0x11 +#define AVS_CMD_S2_ENTER 0x12 +#define AVS_CMD_S2_EXIT 0x13 +#define AVS_CMD_BBM_ENTER 0x14 +#define AVS_CMD_BBM_EXIT 0x15 +#define AVS_CMD_S3_ENTER 0x16 +#define AVS_CMD_S3_EXIT 0x17 +#define AVS_CMD_BALANCE 0x18 +/* PMAP and P-STATE commands */ +#define AVS_CMD_GET_PMAP 0x30 +#define AVS_CMD_SET_PMAP 0x31 +#define AVS_CMD_GET_PSTATE 0x40 +#define AVS_CMD_SET_PSTATE 0x41 + +/* Different modes AVS supports (for GET_PMAP/SET_PMAP) */ +#define AVS_MODE_AVS 0x0 +#define AVS_MODE_DFS 0x1 +#define AVS_MODE_DVS 0x2 +#define AVS_MODE_DVFS 0x3 + +/* + * PMAP parameter p1 + * unused:31-24, mdiv_p0:23-16, unused:15-14, pdiv:13-10 , ndiv_int:9-0 + */ +#define NDIV_INT_SHIFT 0 +#define NDIV_INT_MASK 0x3ff +#define PDIV_SHIFT 10 +#define PDIV_MASK 0xf +#define MDIV_P0_SHIFT 16 +#define MDIV_P0_MASK 0xff +/* + * PMAP parameter p2 + * mdiv_p4:31-24, mdiv_p3:23-16, mdiv_p2:15:8, mdiv_p1:7:0 + */ +#define MDIV_P1_SHIFT 0 +#define MDIV_P1_MASK 0xff +#define MDIV_P2_SHIFT 8 +#define MDIV_P2_MASK 0xff +#define MDIV_P3_SHIFT 16 +#define MDIV_P3_MASK 0xff +#define MDIV_P4_SHIFT 24 +#define MDIV_P4_MASK 0xff + +/* Different P-STATES AVS supports (for GET_PSTATE/SET_PSTATE) */ +#define AVS_PSTATE_P0 0x0 +#define AVS_PSTATE_P1 0x1 +#define AVS_PSTATE_P2 0x2 +#define AVS_PSTATE_P3 0x3 +#define AVS_PSTATE_P4 0x4 +#define AVS_PSTATE_MAX AVS_PSTATE_P4 + +/* CPU L2 Interrupt Controller Registers */ +#define AVS_CPU_L2_SET0 0x04 +#define AVS_CPU_L2_INT_MASK BIT(31) + +/* AVS Command Status Values */ +#define AVS_STATUS_CLEAR 0x00 +/* Command/notification accepted */ +#define AVS_STATUS_SUCCESS 0xf0 +/* Command/notification rejected */ +#define AVS_STATUS_FAILURE 0xff +/* Invalid command/notification (unknown) */ +#define AVS_STATUS_INVALID 0xf1 +/* Non-AVS modes are not supported */ +#define AVS_STATUS_NO_SUPP 0xf2 +/* Cannot set P-State until P-Map supplied */ +#define AVS_STATUS_NO_MAP 0xf3 +/* Cannot change P-Map after initial P-Map set */ +#define AVS_STATUS_MAP_SET 0xf4 +/* Max AVS status; higher numbers are used for debugging */ +#define AVS_STATUS_MAX 0xff + +/* Other AVS related constants */ +#define AVS_LOOP_LIMIT 10000 +#define AVS_TIMEOUT 300 /* in ms; expected completion is < 10ms */ +#define AVS_FIRMWARE_MAGIC 0xa11600d1 + +#define BRCM_AVS_CPUFREQ_PREFIX "brcmstb-avs" +#define BRCM_AVS_CPUFREQ_NAME BRCM_AVS_CPUFREQ_PREFIX "-cpufreq" +#define BRCM_AVS_CPU_DATA "brcm,avs-cpu-data-mem" +#define BRCM_AVS_CPU_INTR "brcm,avs-cpu-l2-intr" +#define BRCM_AVS_HOST_INTR "sw_intr" + +struct pmap { + unsigned int mode; + unsigned int p1; + unsigned int p2; + unsigned int state; +}; + +struct private_data { + void __iomem *base; + void __iomem *avs_intr_base; + struct device *dev; + struct completion done; + struct semaphore sem; + struct pmap pmap; + int host_irq; +}; + +static void __iomem *__map_region(const char *name) +{ + struct device_node *np; + void __iomem *ptr; + + np = of_find_compatible_node(NULL, NULL, name); + if (!np) + return NULL; + + ptr = of_iomap(np, 0); + of_node_put(np); + + return ptr; +} + +static unsigned long wait_for_avs_command(struct private_data *priv, + unsigned long timeout) +{ + unsigned long time_left = 0; + u32 val; + + /* Event driven, wait for the command interrupt */ + if (priv->host_irq >= 0) + return wait_for_completion_timeout(&priv->done, + msecs_to_jiffies(timeout)); + + /* Polling for command completion */ + do { + time_left = timeout; + val = readl(priv->base + AVS_MBOX_STATUS); + if (val) + break; + + usleep_range(1000, 2000); + } while (--timeout); + + return time_left; +} + +static int __issue_avs_command(struct private_data *priv, unsigned int cmd, + unsigned int num_in, unsigned int num_out, + u32 args[]) +{ + void __iomem *base = priv->base; + unsigned long time_left; + unsigned int i; + int ret; + u32 val; + + ret = down_interruptible(&priv->sem); + if (ret) + return ret; + + /* + * Make sure no other command is currently running: cmd is 0 if AVS + * co-processor is idle. Due to the guard above, we should almost never + * have to wait here. + */ + for (i = 0, val = 1; val != 0 && i < AVS_LOOP_LIMIT; i++) + val = readl(base + AVS_MBOX_COMMAND); + + /* Give the caller a chance to retry if AVS is busy. */ + if (i == AVS_LOOP_LIMIT) { + ret = -EAGAIN; + goto out; + } + + /* Clear status before we begin. */ + writel(AVS_STATUS_CLEAR, base + AVS_MBOX_STATUS); + + /* Provide input parameters */ + for (i = 0; i < num_in; i++) + writel(args[i], base + AVS_MBOX_PARAM(i)); + + /* Protect from spurious interrupts. */ + reinit_completion(&priv->done); + + /* Now issue the command & tell firmware to wake up to process it. */ + writel(cmd, base + AVS_MBOX_COMMAND); + writel(AVS_CPU_L2_INT_MASK, priv->avs_intr_base + AVS_CPU_L2_SET0); + + /* Wait for AVS co-processor to finish processing the command. */ + time_left = wait_for_avs_command(priv, AVS_TIMEOUT); + + /* + * If the AVS status is not in the expected range, it means AVS didn't + * complete our command in time, and we return an error. Also, if there + * is no "time left", we timed out waiting for the interrupt. + */ + val = readl(base + AVS_MBOX_STATUS); + if (time_left == 0 || val == 0 || val > AVS_STATUS_MAX) { + dev_err(priv->dev, "AVS command %#x didn't complete in time\n", + cmd); + dev_err(priv->dev, " Time left: %u ms, AVS status: %#x\n", + jiffies_to_msecs(time_left), val); + ret = -ETIMEDOUT; + goto out; + } + + /* Process returned values */ + for (i = 0; i < num_out; i++) + args[i] = readl(base + AVS_MBOX_PARAM(i)); + + /* Clear status to tell AVS co-processor we are done. */ + writel(AVS_STATUS_CLEAR, base + AVS_MBOX_STATUS); + + /* Convert firmware errors to errno's as much as possible. */ + switch (val) { + case AVS_STATUS_INVALID: + ret = -EINVAL; + break; + case AVS_STATUS_NO_SUPP: + ret = -ENOTSUPP; + break; + case AVS_STATUS_NO_MAP: + ret = -ENOENT; + break; + case AVS_STATUS_MAP_SET: + ret = -EEXIST; + break; + case AVS_STATUS_FAILURE: + ret = -EIO; + break; + } + +out: + up(&priv->sem); + + return ret; +} + +static irqreturn_t irq_handler(int irq, void *data) +{ + struct private_data *priv = data; + + /* AVS command completed execution. Wake up __issue_avs_command(). */ + complete(&priv->done); + + return IRQ_HANDLED; +} + +static char *brcm_avs_mode_to_string(unsigned int mode) +{ + switch (mode) { + case AVS_MODE_AVS: + return "AVS"; + case AVS_MODE_DFS: + return "DFS"; + case AVS_MODE_DVS: + return "DVS"; + case AVS_MODE_DVFS: + return "DVFS"; + } + return NULL; +} + +static void brcm_avs_parse_p1(u32 p1, unsigned int *mdiv_p0, unsigned int *pdiv, + unsigned int *ndiv) +{ + *mdiv_p0 = (p1 >> MDIV_P0_SHIFT) & MDIV_P0_MASK; + *pdiv = (p1 >> PDIV_SHIFT) & PDIV_MASK; + *ndiv = (p1 >> NDIV_INT_SHIFT) & NDIV_INT_MASK; +} + +static void brcm_avs_parse_p2(u32 p2, unsigned int *mdiv_p1, + unsigned int *mdiv_p2, unsigned int *mdiv_p3, + unsigned int *mdiv_p4) +{ + *mdiv_p4 = (p2 >> MDIV_P4_SHIFT) & MDIV_P4_MASK; + *mdiv_p3 = (p2 >> MDIV_P3_SHIFT) & MDIV_P3_MASK; + *mdiv_p2 = (p2 >> MDIV_P2_SHIFT) & MDIV_P2_MASK; + *mdiv_p1 = (p2 >> MDIV_P1_SHIFT) & MDIV_P1_MASK; +} + +static int brcm_avs_get_pmap(struct private_data *priv, struct pmap *pmap) +{ + u32 args[AVS_MAX_CMD_ARGS]; + int ret; + + ret = __issue_avs_command(priv, AVS_CMD_GET_PMAP, 0, 4, args); + if (ret || !pmap) + return ret; + + pmap->mode = args[0]; + pmap->p1 = args[1]; + pmap->p2 = args[2]; + pmap->state = args[3]; + + return 0; +} + +static int brcm_avs_set_pmap(struct private_data *priv, struct pmap *pmap) +{ + u32 args[AVS_MAX_CMD_ARGS]; + + args[0] = pmap->mode; + args[1] = pmap->p1; + args[2] = pmap->p2; + args[3] = pmap->state; + + return __issue_avs_command(priv, AVS_CMD_SET_PMAP, 4, 0, args); +} + +static int brcm_avs_get_pstate(struct private_data *priv, unsigned int *pstate) +{ + u32 args[AVS_MAX_CMD_ARGS]; + int ret; + + ret = __issue_avs_command(priv, AVS_CMD_GET_PSTATE, 0, 1, args); + if (ret) + return ret; + *pstate = args[0]; + + return 0; +} + +static int brcm_avs_set_pstate(struct private_data *priv, unsigned int pstate) +{ + u32 args[AVS_MAX_CMD_ARGS]; + + args[0] = pstate; + + return __issue_avs_command(priv, AVS_CMD_SET_PSTATE, 1, 0, args); + +} + +static u32 brcm_avs_get_voltage(void __iomem *base) +{ + return readl(base + AVS_MBOX_VOLTAGE1); +} + +static u32 brcm_avs_get_frequency(void __iomem *base) +{ + return readl(base + AVS_MBOX_FREQUENCY) * 1000; /* in kHz */ +} + +/* + * We determine which frequencies are supported by cycling through all P-states + * and reading back what frequency we are running at for each P-state. + */ +static struct cpufreq_frequency_table * +brcm_avs_get_freq_table(struct device *dev, struct private_data *priv) +{ + struct cpufreq_frequency_table *table; + unsigned int pstate; + int i, ret; + + /* Remember P-state for later */ + ret = brcm_avs_get_pstate(priv, &pstate); + if (ret) + return ERR_PTR(ret); + + /* + * We allocate space for the 5 different P-STATES AVS, + * plus extra space for a terminating element. + */ + table = devm_kcalloc(dev, AVS_PSTATE_MAX + 1 + 1, sizeof(*table), + GFP_KERNEL); + if (!table) + return ERR_PTR(-ENOMEM); + + for (i = AVS_PSTATE_P0; i <= AVS_PSTATE_MAX; i++) { + ret = brcm_avs_set_pstate(priv, i); + if (ret) + return ERR_PTR(ret); + table[i].frequency = brcm_avs_get_frequency(priv->base); + table[i].driver_data = i; + } + table[i].frequency = CPUFREQ_TABLE_END; + + /* Restore P-state */ + ret = brcm_avs_set_pstate(priv, pstate); + if (ret) + return ERR_PTR(ret); + + return table; +} + +/* + * To ensure the right firmware is running we need to + * - check the MAGIC matches what we expect + * - brcm_avs_get_pmap() doesn't return -ENOTSUPP or -EINVAL + * We need to set up our interrupt handling before calling brcm_avs_get_pmap()! + */ +static bool brcm_avs_is_firmware_loaded(struct private_data *priv) +{ + u32 magic; + int rc; + + rc = brcm_avs_get_pmap(priv, NULL); + magic = readl(priv->base + AVS_MBOX_MAGIC); + + return (magic == AVS_FIRMWARE_MAGIC) && (rc != -ENOTSUPP) && + (rc != -EINVAL); +} + +static unsigned int brcm_avs_cpufreq_get(unsigned int cpu) +{ + struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu); + struct private_data *priv; + + if (!policy) + return 0; + + priv = policy->driver_data; + + return brcm_avs_get_frequency(priv->base); +} + +static int brcm_avs_target_index(struct cpufreq_policy *policy, + unsigned int index) +{ + return brcm_avs_set_pstate(policy->driver_data, + policy->freq_table[index].driver_data); +} + +static int brcm_avs_suspend(struct cpufreq_policy *policy) +{ + struct private_data *priv = policy->driver_data; + int ret; + + ret = brcm_avs_get_pmap(priv, &priv->pmap); + if (ret) + return ret; + + /* + * We can't use the P-state returned by brcm_avs_get_pmap(), since + * that's the initial P-state from when the P-map was downloaded to the + * AVS co-processor, not necessarily the P-state we are running at now. + * So, we get the current P-state explicitly. + */ + ret = brcm_avs_get_pstate(priv, &priv->pmap.state); + if (ret) + return ret; + + /* This is best effort. Nothing to do if it fails. */ + (void)__issue_avs_command(priv, AVS_CMD_S2_ENTER, 0, 0, NULL); + + return 0; +} + +static int brcm_avs_resume(struct cpufreq_policy *policy) +{ + struct private_data *priv = policy->driver_data; + int ret; + + /* This is best effort. Nothing to do if it fails. */ + (void)__issue_avs_command(priv, AVS_CMD_S2_EXIT, 0, 0, NULL); + + ret = brcm_avs_set_pmap(priv, &priv->pmap); + if (ret == -EEXIST) { + struct platform_device *pdev = cpufreq_get_driver_data(); + struct device *dev = &pdev->dev; + + dev_warn(dev, "PMAP was already set\n"); + ret = 0; + } + + return ret; +} + +/* + * All initialization code that we only want to execute once goes here. Setup + * code that can be re-tried on every core (if it failed before) can go into + * brcm_avs_cpufreq_init(). + */ +static int brcm_avs_prepare_init(struct platform_device *pdev) +{ + struct private_data *priv; + struct device *dev; + int ret; + + dev = &pdev->dev; + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + priv->dev = dev; + sema_init(&priv->sem, 1); + init_completion(&priv->done); + platform_set_drvdata(pdev, priv); + + priv->base = __map_region(BRCM_AVS_CPU_DATA); + if (!priv->base) { + dev_err(dev, "Couldn't find property %s in device tree.\n", + BRCM_AVS_CPU_DATA); + return -ENOENT; + } + + priv->avs_intr_base = __map_region(BRCM_AVS_CPU_INTR); + if (!priv->avs_intr_base) { + dev_err(dev, "Couldn't find property %s in device tree.\n", + BRCM_AVS_CPU_INTR); + ret = -ENOENT; + goto unmap_base; + } + + priv->host_irq = platform_get_irq_byname(pdev, BRCM_AVS_HOST_INTR); + + ret = devm_request_irq(dev, priv->host_irq, irq_handler, + IRQF_TRIGGER_RISING, + BRCM_AVS_HOST_INTR, priv); + if (ret && priv->host_irq >= 0) { + dev_err(dev, "IRQ request failed: %s (%d) -- %d\n", + BRCM_AVS_HOST_INTR, priv->host_irq, ret); + goto unmap_intr_base; + } + + if (brcm_avs_is_firmware_loaded(priv)) + return 0; + + dev_err(dev, "AVS firmware is not loaded or doesn't support DVFS\n"); + ret = -ENODEV; + +unmap_intr_base: + iounmap(priv->avs_intr_base); +unmap_base: + iounmap(priv->base); + + return ret; +} + +static void brcm_avs_prepare_uninit(struct platform_device *pdev) +{ + struct private_data *priv; + + priv = platform_get_drvdata(pdev); + + iounmap(priv->avs_intr_base); + iounmap(priv->base); +} + +static int brcm_avs_cpufreq_init(struct cpufreq_policy *policy) +{ + struct cpufreq_frequency_table *freq_table; + struct platform_device *pdev; + struct private_data *priv; + struct device *dev; + int ret; + + pdev = cpufreq_get_driver_data(); + priv = platform_get_drvdata(pdev); + policy->driver_data = priv; + dev = &pdev->dev; + + freq_table = brcm_avs_get_freq_table(dev, priv); + if (IS_ERR(freq_table)) { + ret = PTR_ERR(freq_table); + dev_err(dev, "Couldn't determine frequency table (%d).\n", ret); + return ret; + } + + policy->freq_table = freq_table; + + /* All cores share the same clock and thus the same policy. */ + cpumask_setall(policy->cpus); + + ret = __issue_avs_command(priv, AVS_CMD_ENABLE, 0, 0, NULL); + if (!ret) { + unsigned int pstate; + + ret = brcm_avs_get_pstate(priv, &pstate); + if (!ret) { + policy->cur = freq_table[pstate].frequency; + dev_info(dev, "registered\n"); + return 0; + } + } + + dev_err(dev, "couldn't initialize driver (%d)\n", ret); + + return ret; +} + +static ssize_t show_brcm_avs_pstate(struct cpufreq_policy *policy, char *buf) +{ + struct private_data *priv = policy->driver_data; + unsigned int pstate; + + if (brcm_avs_get_pstate(priv, &pstate)) + return sprintf(buf, "<unknown>\n"); + + return sprintf(buf, "%u\n", pstate); +} + +static ssize_t show_brcm_avs_mode(struct cpufreq_policy *policy, char *buf) +{ + struct private_data *priv = policy->driver_data; + struct pmap pmap; + + if (brcm_avs_get_pmap(priv, &pmap)) + return sprintf(buf, "<unknown>\n"); + + return sprintf(buf, "%s %u\n", brcm_avs_mode_to_string(pmap.mode), + pmap.mode); +} + +static ssize_t show_brcm_avs_pmap(struct cpufreq_policy *policy, char *buf) +{ + unsigned int mdiv_p0, mdiv_p1, mdiv_p2, mdiv_p3, mdiv_p4; + struct private_data *priv = policy->driver_data; + unsigned int ndiv, pdiv; + struct pmap pmap; + + if (brcm_avs_get_pmap(priv, &pmap)) + return sprintf(buf, "<unknown>\n"); + + brcm_avs_parse_p1(pmap.p1, &mdiv_p0, &pdiv, &ndiv); + brcm_avs_parse_p2(pmap.p2, &mdiv_p1, &mdiv_p2, &mdiv_p3, &mdiv_p4); + + return sprintf(buf, "0x%08x 0x%08x %u %u %u %u %u %u %u %u %u\n", + pmap.p1, pmap.p2, ndiv, pdiv, mdiv_p0, mdiv_p1, mdiv_p2, + mdiv_p3, mdiv_p4, pmap.mode, pmap.state); +} + +static ssize_t show_brcm_avs_voltage(struct cpufreq_policy *policy, char *buf) +{ + struct private_data *priv = policy->driver_data; + + return sprintf(buf, "0x%08x\n", brcm_avs_get_voltage(priv->base)); +} + +static ssize_t show_brcm_avs_frequency(struct cpufreq_policy *policy, char *buf) +{ + struct private_data *priv = policy->driver_data; + + return sprintf(buf, "0x%08x\n", brcm_avs_get_frequency(priv->base)); +} + +cpufreq_freq_attr_ro(brcm_avs_pstate); +cpufreq_freq_attr_ro(brcm_avs_mode); +cpufreq_freq_attr_ro(brcm_avs_pmap); +cpufreq_freq_attr_ro(brcm_avs_voltage); +cpufreq_freq_attr_ro(brcm_avs_frequency); + +static struct freq_attr *brcm_avs_cpufreq_attr[] = { + &brcm_avs_pstate, + &brcm_avs_mode, + &brcm_avs_pmap, + &brcm_avs_voltage, + &brcm_avs_frequency, + NULL +}; + +static struct cpufreq_driver brcm_avs_driver = { + .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK, + .verify = cpufreq_generic_frequency_table_verify, + .target_index = brcm_avs_target_index, + .get = brcm_avs_cpufreq_get, + .suspend = brcm_avs_suspend, + .resume = brcm_avs_resume, + .init = brcm_avs_cpufreq_init, + .attr = brcm_avs_cpufreq_attr, + .name = BRCM_AVS_CPUFREQ_PREFIX, +}; + +static int brcm_avs_cpufreq_probe(struct platform_device *pdev) +{ + int ret; + + ret = brcm_avs_prepare_init(pdev); + if (ret) + return ret; + + brcm_avs_driver.driver_data = pdev; + + ret = cpufreq_register_driver(&brcm_avs_driver); + if (ret) + brcm_avs_prepare_uninit(pdev); + + return ret; +} + +static void brcm_avs_cpufreq_remove(struct platform_device *pdev) +{ + cpufreq_unregister_driver(&brcm_avs_driver); + + brcm_avs_prepare_uninit(pdev); +} + +static const struct of_device_id brcm_avs_cpufreq_match[] = { + { .compatible = "brcm,avs-cpu-data-mem" }, + { } +}; +MODULE_DEVICE_TABLE(of, brcm_avs_cpufreq_match); + +static struct platform_driver brcm_avs_cpufreq_platdrv = { + .driver = { + .name = BRCM_AVS_CPUFREQ_NAME, + .of_match_table = brcm_avs_cpufreq_match, + }, + .probe = brcm_avs_cpufreq_probe, + .remove = brcm_avs_cpufreq_remove, +}; +module_platform_driver(brcm_avs_cpufreq_platdrv); + +MODULE_AUTHOR("Markus Mayer <mmayer@broadcom.com>"); +MODULE_DESCRIPTION("CPUfreq driver for Broadcom STB AVS"); +MODULE_LICENSE("GPL"); diff --git a/drivers/cpufreq/cppc_cpufreq.c b/drivers/cpufreq/cppc_cpufreq.c new file mode 100644 index 000000000000..9eac77c4f294 --- /dev/null +++ b/drivers/cpufreq/cppc_cpufreq.c @@ -0,0 +1,963 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * CPPC (Collaborative Processor Performance Control) driver for + * interfacing with the CPUfreq layer and governors. See + * cppc_acpi.c for CPPC specific methods. + * + * (C) Copyright 2014, 2015 Linaro Ltd. + * Author: Ashwin Chaugule <ashwin.chaugule@linaro.org> + */ + +#define pr_fmt(fmt) "CPPC Cpufreq:" fmt + +#include <linux/arch_topology.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/delay.h> +#include <linux/cpu.h> +#include <linux/cpufreq.h> +#include <linux/irq_work.h> +#include <linux/kthread.h> +#include <linux/time.h> +#include <linux/vmalloc.h> +#include <uapi/linux/sched/types.h> + +#include <linux/unaligned.h> + +#include <acpi/cppc_acpi.h> + +static struct cpufreq_driver cppc_cpufreq_driver; + +#ifdef CONFIG_ACPI_CPPC_CPUFREQ_FIE +static enum { + FIE_UNSET = -1, + FIE_ENABLED, + FIE_DISABLED +} fie_disabled = FIE_UNSET; + +module_param(fie_disabled, int, 0444); +MODULE_PARM_DESC(fie_disabled, "Disable Frequency Invariance Engine (FIE)"); + +/* Frequency invariance support */ +struct cppc_freq_invariance { + int cpu; + struct irq_work irq_work; + struct kthread_work work; + struct cppc_perf_fb_ctrs prev_perf_fb_ctrs; + struct cppc_cpudata *cpu_data; +}; + +static DEFINE_PER_CPU(struct cppc_freq_invariance, cppc_freq_inv); +static struct kthread_worker *kworker_fie; + +static int cppc_perf_from_fbctrs(struct cppc_perf_fb_ctrs *fb_ctrs_t0, + struct cppc_perf_fb_ctrs *fb_ctrs_t1); + +/** + * cppc_scale_freq_workfn - CPPC arch_freq_scale updater for frequency invariance + * @work: The work item. + * + * The CPPC driver register itself with the topology core to provide its own + * implementation (cppc_scale_freq_tick()) of topology_scale_freq_tick() which + * gets called by the scheduler on every tick. + * + * Note that the arch specific counters have higher priority than CPPC counters, + * if available, though the CPPC driver doesn't need to have any special + * handling for that. + * + * On an invocation of cppc_scale_freq_tick(), we schedule an irq work (since we + * reach here from hard-irq context), which then schedules a normal work item + * and cppc_scale_freq_workfn() updates the per_cpu arch_freq_scale variable + * based on the counter updates since the last tick. + */ +static void cppc_scale_freq_workfn(struct kthread_work *work) +{ + struct cppc_freq_invariance *cppc_fi; + struct cppc_perf_fb_ctrs fb_ctrs = {0}; + struct cppc_cpudata *cpu_data; + unsigned long local_freq_scale; + u64 perf; + + cppc_fi = container_of(work, struct cppc_freq_invariance, work); + cpu_data = cppc_fi->cpu_data; + + if (cppc_get_perf_ctrs(cppc_fi->cpu, &fb_ctrs)) { + pr_warn("%s: failed to read perf counters\n", __func__); + return; + } + + perf = cppc_perf_from_fbctrs(&cppc_fi->prev_perf_fb_ctrs, &fb_ctrs); + if (!perf) + return; + + cppc_fi->prev_perf_fb_ctrs = fb_ctrs; + + perf <<= SCHED_CAPACITY_SHIFT; + local_freq_scale = div64_u64(perf, cpu_data->perf_caps.highest_perf); + + /* This can happen due to counter's overflow */ + if (unlikely(local_freq_scale > 1024)) + local_freq_scale = 1024; + + per_cpu(arch_freq_scale, cppc_fi->cpu) = local_freq_scale; +} + +static void cppc_irq_work(struct irq_work *irq_work) +{ + struct cppc_freq_invariance *cppc_fi; + + cppc_fi = container_of(irq_work, struct cppc_freq_invariance, irq_work); + kthread_queue_work(kworker_fie, &cppc_fi->work); +} + +static void cppc_scale_freq_tick(void) +{ + struct cppc_freq_invariance *cppc_fi = &per_cpu(cppc_freq_inv, smp_processor_id()); + + /* + * cppc_get_perf_ctrs() can potentially sleep, call that from the right + * context. + */ + irq_work_queue(&cppc_fi->irq_work); +} + +static struct scale_freq_data cppc_sftd = { + .source = SCALE_FREQ_SOURCE_CPPC, + .set_freq_scale = cppc_scale_freq_tick, +}; + +static void cppc_cpufreq_cpu_fie_init(struct cpufreq_policy *policy) +{ + struct cppc_freq_invariance *cppc_fi; + int cpu, ret; + + if (fie_disabled) + return; + + for_each_cpu(cpu, policy->cpus) { + cppc_fi = &per_cpu(cppc_freq_inv, cpu); + cppc_fi->cpu = cpu; + cppc_fi->cpu_data = policy->driver_data; + kthread_init_work(&cppc_fi->work, cppc_scale_freq_workfn); + init_irq_work(&cppc_fi->irq_work, cppc_irq_work); + + ret = cppc_get_perf_ctrs(cpu, &cppc_fi->prev_perf_fb_ctrs); + + /* + * Don't abort as the CPU was offline while the driver was + * getting registered. + */ + if (ret && cpu_online(cpu)) { + pr_debug("%s: failed to read perf counters for cpu:%d: %d\n", + __func__, cpu, ret); + return; + } + } + + /* Register for freq-invariance */ + topology_set_scale_freq_source(&cppc_sftd, policy->cpus); +} + +/* + * We free all the resources on policy's removal and not on CPU removal as the + * irq-work are per-cpu and the hotplug core takes care of flushing the pending + * irq-works (hint: smpcfd_dying_cpu()) on CPU hotplug. Even if the kthread-work + * fires on another CPU after the concerned CPU is removed, it won't harm. + * + * We just need to make sure to remove them all on policy->exit(). + */ +static void cppc_cpufreq_cpu_fie_exit(struct cpufreq_policy *policy) +{ + struct cppc_freq_invariance *cppc_fi; + int cpu; + + if (fie_disabled) + return; + + /* policy->cpus will be empty here, use related_cpus instead */ + topology_clear_scale_freq_source(SCALE_FREQ_SOURCE_CPPC, policy->related_cpus); + + for_each_cpu(cpu, policy->related_cpus) { + cppc_fi = &per_cpu(cppc_freq_inv, cpu); + irq_work_sync(&cppc_fi->irq_work); + kthread_cancel_work_sync(&cppc_fi->work); + } +} + +static void __init cppc_freq_invariance_init(void) +{ + struct sched_attr attr = { + .size = sizeof(struct sched_attr), + .sched_policy = SCHED_DEADLINE, + .sched_nice = 0, + .sched_priority = 0, + /* + * Fake (unused) bandwidth; workaround to "fix" + * priority inheritance. + */ + .sched_runtime = NSEC_PER_MSEC, + .sched_deadline = 10 * NSEC_PER_MSEC, + .sched_period = 10 * NSEC_PER_MSEC, + }; + int ret; + + if (fie_disabled != FIE_ENABLED && fie_disabled != FIE_DISABLED) { + fie_disabled = FIE_ENABLED; + if (cppc_perf_ctrs_in_pcc()) { + pr_info("FIE not enabled on systems with registers in PCC\n"); + fie_disabled = FIE_DISABLED; + } + } + + if (fie_disabled) + return; + + kworker_fie = kthread_run_worker(0, "cppc_fie"); + if (IS_ERR(kworker_fie)) { + pr_warn("%s: failed to create kworker_fie: %ld\n", __func__, + PTR_ERR(kworker_fie)); + fie_disabled = FIE_DISABLED; + return; + } + + ret = sched_setattr_nocheck(kworker_fie->task, &attr); + if (ret) { + pr_warn("%s: failed to set SCHED_DEADLINE: %d\n", __func__, + ret); + kthread_destroy_worker(kworker_fie); + fie_disabled = FIE_DISABLED; + } +} + +static void cppc_freq_invariance_exit(void) +{ + if (fie_disabled) + return; + + kthread_destroy_worker(kworker_fie); +} + +#else +static inline void cppc_cpufreq_cpu_fie_init(struct cpufreq_policy *policy) +{ +} + +static inline void cppc_cpufreq_cpu_fie_exit(struct cpufreq_policy *policy) +{ +} + +static inline void cppc_freq_invariance_init(void) +{ +} + +static inline void cppc_freq_invariance_exit(void) +{ +} +#endif /* CONFIG_ACPI_CPPC_CPUFREQ_FIE */ + +static int cppc_cpufreq_set_target(struct cpufreq_policy *policy, + unsigned int target_freq, + unsigned int relation) +{ + struct cppc_cpudata *cpu_data = policy->driver_data; + unsigned int cpu = policy->cpu; + struct cpufreq_freqs freqs; + int ret = 0; + + cpu_data->perf_ctrls.desired_perf = + cppc_khz_to_perf(&cpu_data->perf_caps, target_freq); + freqs.old = policy->cur; + freqs.new = target_freq; + + cpufreq_freq_transition_begin(policy, &freqs); + ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls); + cpufreq_freq_transition_end(policy, &freqs, ret != 0); + + if (ret) + pr_debug("Failed to set target on CPU:%d. ret:%d\n", + cpu, ret); + + return ret; +} + +static unsigned int cppc_cpufreq_fast_switch(struct cpufreq_policy *policy, + unsigned int target_freq) +{ + struct cppc_cpudata *cpu_data = policy->driver_data; + unsigned int cpu = policy->cpu; + u32 desired_perf; + int ret; + + desired_perf = cppc_khz_to_perf(&cpu_data->perf_caps, target_freq); + cpu_data->perf_ctrls.desired_perf = desired_perf; + ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls); + + if (ret) { + pr_debug("Failed to set target on CPU:%d. ret:%d\n", + cpu, ret); + return 0; + } + + return target_freq; +} + +static int cppc_verify_policy(struct cpufreq_policy_data *policy) +{ + cpufreq_verify_within_cpu_limits(policy); + return 0; +} + +static unsigned int __cppc_cpufreq_get_transition_delay_us(unsigned int cpu) +{ + int transition_latency_ns = cppc_get_transition_latency(cpu); + + if (transition_latency_ns < 0) + return CPUFREQ_DEFAULT_TRANSITION_LATENCY_NS / NSEC_PER_USEC; + + return transition_latency_ns / NSEC_PER_USEC; +} + +/* + * The PCC subspace describes the rate at which platform can accept commands + * on the shared PCC channel (including READs which do not count towards freq + * transition requests), so ideally we need to use the PCC values as a fallback + * if we don't have a platform specific transition_delay_us + */ +#ifdef CONFIG_ARM64 +#include <asm/cputype.h> + +static unsigned int cppc_cpufreq_get_transition_delay_us(unsigned int cpu) +{ + unsigned long implementor = read_cpuid_implementor(); + unsigned long part_num = read_cpuid_part_number(); + + switch (implementor) { + case ARM_CPU_IMP_QCOM: + switch (part_num) { + case QCOM_CPU_PART_FALKOR_V1: + case QCOM_CPU_PART_FALKOR: + return 10000; + } + } + return __cppc_cpufreq_get_transition_delay_us(cpu); +} +#else +static unsigned int cppc_cpufreq_get_transition_delay_us(unsigned int cpu) +{ + return __cppc_cpufreq_get_transition_delay_us(cpu); +} +#endif + +#if defined(CONFIG_ARM64) && defined(CONFIG_ENERGY_MODEL) + +static DEFINE_PER_CPU(unsigned int, efficiency_class); + +/* Create an artificial performance state every CPPC_EM_CAP_STEP capacity unit. */ +#define CPPC_EM_CAP_STEP (20) +/* Increase the cost value by CPPC_EM_COST_STEP every performance state. */ +#define CPPC_EM_COST_STEP (1) +/* Add a cost gap correspnding to the energy of 4 CPUs. */ +#define CPPC_EM_COST_GAP (4 * SCHED_CAPACITY_SCALE * CPPC_EM_COST_STEP \ + / CPPC_EM_CAP_STEP) + +static unsigned int get_perf_level_count(struct cpufreq_policy *policy) +{ + struct cppc_perf_caps *perf_caps; + unsigned int min_cap, max_cap; + struct cppc_cpudata *cpu_data; + int cpu = policy->cpu; + + cpu_data = policy->driver_data; + perf_caps = &cpu_data->perf_caps; + max_cap = arch_scale_cpu_capacity(cpu); + min_cap = div_u64((u64)max_cap * perf_caps->lowest_perf, + perf_caps->highest_perf); + if ((min_cap == 0) || (max_cap < min_cap)) + return 0; + return 1 + max_cap / CPPC_EM_CAP_STEP - min_cap / CPPC_EM_CAP_STEP; +} + +/* + * The cost is defined as: + * cost = power * max_frequency / frequency + */ +static inline unsigned long compute_cost(int cpu, int step) +{ + return CPPC_EM_COST_GAP * per_cpu(efficiency_class, cpu) + + step * CPPC_EM_COST_STEP; +} + +static int cppc_get_cpu_power(struct device *cpu_dev, + unsigned long *power, unsigned long *KHz) +{ + unsigned long perf_step, perf_prev, perf, perf_check; + unsigned int min_step, max_step, step, step_check; + unsigned long prev_freq = *KHz; + unsigned int min_cap, max_cap; + struct cpufreq_policy *policy; + + struct cppc_perf_caps *perf_caps; + struct cppc_cpudata *cpu_data; + + policy = cpufreq_cpu_get_raw(cpu_dev->id); + if (!policy) + return -EINVAL; + + cpu_data = policy->driver_data; + perf_caps = &cpu_data->perf_caps; + max_cap = arch_scale_cpu_capacity(cpu_dev->id); + min_cap = div_u64((u64)max_cap * perf_caps->lowest_perf, + perf_caps->highest_perf); + perf_step = div_u64((u64)CPPC_EM_CAP_STEP * perf_caps->highest_perf, + max_cap); + min_step = min_cap / CPPC_EM_CAP_STEP; + max_step = max_cap / CPPC_EM_CAP_STEP; + + perf_prev = cppc_khz_to_perf(perf_caps, *KHz); + step = perf_prev / perf_step; + + if (step > max_step) + return -EINVAL; + + if (min_step == max_step) { + step = max_step; + perf = perf_caps->highest_perf; + } else if (step < min_step) { + step = min_step; + perf = perf_caps->lowest_perf; + } else { + step++; + if (step == max_step) + perf = perf_caps->highest_perf; + else + perf = step * perf_step; + } + + *KHz = cppc_perf_to_khz(perf_caps, perf); + perf_check = cppc_khz_to_perf(perf_caps, *KHz); + step_check = perf_check / perf_step; + + /* + * To avoid bad integer approximation, check that new frequency value + * increased and that the new frequency will be converted to the + * desired step value. + */ + while ((*KHz == prev_freq) || (step_check != step)) { + perf++; + *KHz = cppc_perf_to_khz(perf_caps, perf); + perf_check = cppc_khz_to_perf(perf_caps, *KHz); + step_check = perf_check / perf_step; + } + + /* + * With an artificial EM, only the cost value is used. Still the power + * is populated such as 0 < power < EM_MAX_POWER. This allows to add + * more sense to the artificial performance states. + */ + *power = compute_cost(cpu_dev->id, step); + + return 0; +} + +static int cppc_get_cpu_cost(struct device *cpu_dev, unsigned long KHz, + unsigned long *cost) +{ + unsigned long perf_step, perf_prev; + struct cppc_perf_caps *perf_caps; + struct cpufreq_policy *policy; + struct cppc_cpudata *cpu_data; + unsigned int max_cap; + int step; + + policy = cpufreq_cpu_get_raw(cpu_dev->id); + if (!policy) + return -EINVAL; + + cpu_data = policy->driver_data; + perf_caps = &cpu_data->perf_caps; + max_cap = arch_scale_cpu_capacity(cpu_dev->id); + + perf_prev = cppc_khz_to_perf(perf_caps, KHz); + perf_step = CPPC_EM_CAP_STEP * perf_caps->highest_perf / max_cap; + step = perf_prev / perf_step; + + *cost = compute_cost(cpu_dev->id, step); + + return 0; +} + +static void cppc_cpufreq_register_em(struct cpufreq_policy *policy) +{ + struct cppc_cpudata *cpu_data; + struct em_data_callback em_cb = + EM_ADV_DATA_CB(cppc_get_cpu_power, cppc_get_cpu_cost); + + cpu_data = policy->driver_data; + em_dev_register_perf_domain(get_cpu_device(policy->cpu), + get_perf_level_count(policy), &em_cb, + cpu_data->shared_cpu_map, 0); +} + +static void populate_efficiency_class(void) +{ + struct acpi_madt_generic_interrupt *gicc; + DECLARE_BITMAP(used_classes, 256) = {}; + int class, cpu, index; + + for_each_possible_cpu(cpu) { + gicc = acpi_cpu_get_madt_gicc(cpu); + class = gicc->efficiency_class; + bitmap_set(used_classes, class, 1); + } + + if (bitmap_weight(used_classes, 256) <= 1) { + pr_debug("Efficiency classes are all equal (=%d). " + "No EM registered", class); + return; + } + + /* + * Squeeze efficiency class values on [0:#efficiency_class-1]. + * Values are per spec in [0:255]. + */ + index = 0; + for_each_set_bit(class, used_classes, 256) { + for_each_possible_cpu(cpu) { + gicc = acpi_cpu_get_madt_gicc(cpu); + if (gicc->efficiency_class == class) + per_cpu(efficiency_class, cpu) = index; + } + index++; + } + cppc_cpufreq_driver.register_em = cppc_cpufreq_register_em; +} + +#else +static void populate_efficiency_class(void) +{ +} +#endif + +static struct cppc_cpudata *cppc_cpufreq_get_cpu_data(unsigned int cpu) +{ + struct cppc_cpudata *cpu_data; + int ret; + + cpu_data = kzalloc(sizeof(struct cppc_cpudata), GFP_KERNEL); + if (!cpu_data) + goto out; + + if (!zalloc_cpumask_var(&cpu_data->shared_cpu_map, GFP_KERNEL)) + goto free_cpu; + + ret = acpi_get_psd_map(cpu, cpu_data); + if (ret) { + pr_debug("Err parsing CPU%d PSD data: ret:%d\n", cpu, ret); + goto free_mask; + } + + ret = cppc_get_perf_caps(cpu, &cpu_data->perf_caps); + if (ret) { + pr_debug("Err reading CPU%d perf caps: ret:%d\n", cpu, ret); + goto free_mask; + } + + return cpu_data; + +free_mask: + free_cpumask_var(cpu_data->shared_cpu_map); +free_cpu: + kfree(cpu_data); +out: + return NULL; +} + +static void cppc_cpufreq_put_cpu_data(struct cpufreq_policy *policy) +{ + struct cppc_cpudata *cpu_data = policy->driver_data; + + free_cpumask_var(cpu_data->shared_cpu_map); + kfree(cpu_data); + policy->driver_data = NULL; +} + +static int cppc_cpufreq_cpu_init(struct cpufreq_policy *policy) +{ + unsigned int cpu = policy->cpu; + struct cppc_cpudata *cpu_data; + struct cppc_perf_caps *caps; + int ret; + + cpu_data = cppc_cpufreq_get_cpu_data(cpu); + if (!cpu_data) { + pr_err("Error in acquiring _CPC/_PSD data for CPU%d.\n", cpu); + return -ENODEV; + } + caps = &cpu_data->perf_caps; + policy->driver_data = cpu_data; + + /* + * Set min to lowest nonlinear perf to avoid any efficiency penalty (see + * Section 8.4.7.1.1.5 of ACPI 6.1 spec) + */ + policy->min = cppc_perf_to_khz(caps, caps->lowest_nonlinear_perf); + policy->max = cppc_perf_to_khz(caps, policy->boost_enabled ? + caps->highest_perf : caps->nominal_perf); + + /* + * Set cpuinfo.min_freq to Lowest to make the full range of performance + * available if userspace wants to use any perf between lowest & lowest + * nonlinear perf + */ + policy->cpuinfo.min_freq = cppc_perf_to_khz(caps, caps->lowest_perf); + policy->cpuinfo.max_freq = policy->max; + + policy->transition_delay_us = cppc_cpufreq_get_transition_delay_us(cpu); + policy->shared_type = cpu_data->shared_type; + + switch (policy->shared_type) { + case CPUFREQ_SHARED_TYPE_HW: + case CPUFREQ_SHARED_TYPE_NONE: + /* Nothing to be done - we'll have a policy for each CPU */ + break; + case CPUFREQ_SHARED_TYPE_ANY: + /* + * All CPUs in the domain will share a policy and all cpufreq + * operations will use a single cppc_cpudata structure stored + * in policy->driver_data. + */ + cpumask_copy(policy->cpus, cpu_data->shared_cpu_map); + break; + default: + pr_debug("Unsupported CPU co-ord type: %d\n", + policy->shared_type); + ret = -EFAULT; + goto out; + } + + policy->fast_switch_possible = cppc_allow_fast_switch(); + policy->dvfs_possible_from_any_cpu = true; + + /* + * If 'highest_perf' is greater than 'nominal_perf', we assume CPU Boost + * is supported. + */ + if (caps->highest_perf > caps->nominal_perf) + policy->boost_supported = true; + + /* Set policy->cur to max now. The governors will adjust later. */ + policy->cur = cppc_perf_to_khz(caps, caps->highest_perf); + cpu_data->perf_ctrls.desired_perf = caps->highest_perf; + + ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls); + if (ret) { + pr_debug("Err setting perf value:%d on CPU:%d. ret:%d\n", + caps->highest_perf, cpu, ret); + goto out; + } + + cppc_cpufreq_cpu_fie_init(policy); + return 0; + +out: + cppc_cpufreq_put_cpu_data(policy); + return ret; +} + +static void cppc_cpufreq_cpu_exit(struct cpufreq_policy *policy) +{ + struct cppc_cpudata *cpu_data = policy->driver_data; + struct cppc_perf_caps *caps = &cpu_data->perf_caps; + unsigned int cpu = policy->cpu; + int ret; + + cppc_cpufreq_cpu_fie_exit(policy); + + cpu_data->perf_ctrls.desired_perf = caps->lowest_perf; + + ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls); + if (ret) + pr_debug("Err setting perf value:%d on CPU:%d. ret:%d\n", + caps->lowest_perf, cpu, ret); + + cppc_cpufreq_put_cpu_data(policy); +} + +static inline u64 get_delta(u64 t1, u64 t0) +{ + if (t1 > t0 || t0 > ~(u32)0) + return t1 - t0; + + return (u32)t1 - (u32)t0; +} + +static int cppc_perf_from_fbctrs(struct cppc_perf_fb_ctrs *fb_ctrs_t0, + struct cppc_perf_fb_ctrs *fb_ctrs_t1) +{ + u64 delta_reference, delta_delivered; + u64 reference_perf; + + reference_perf = fb_ctrs_t0->reference_perf; + + delta_reference = get_delta(fb_ctrs_t1->reference, + fb_ctrs_t0->reference); + delta_delivered = get_delta(fb_ctrs_t1->delivered, + fb_ctrs_t0->delivered); + + /* + * Avoid divide-by zero and unchanged feedback counters. + * Leave it for callers to handle. + */ + if (!delta_reference || !delta_delivered) + return 0; + + return (reference_perf * delta_delivered) / delta_reference; +} + +static int cppc_get_perf_ctrs_sample(int cpu, + struct cppc_perf_fb_ctrs *fb_ctrs_t0, + struct cppc_perf_fb_ctrs *fb_ctrs_t1) +{ + int ret; + + ret = cppc_get_perf_ctrs(cpu, fb_ctrs_t0); + if (ret) + return ret; + + udelay(2); /* 2usec delay between sampling */ + + return cppc_get_perf_ctrs(cpu, fb_ctrs_t1); +} + +static unsigned int cppc_cpufreq_get_rate(unsigned int cpu) +{ + struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu); + struct cppc_perf_fb_ctrs fb_ctrs_t0 = {0}, fb_ctrs_t1 = {0}; + struct cppc_cpudata *cpu_data; + u64 delivered_perf; + int ret; + + if (!policy) + return 0; + + cpu_data = policy->driver_data; + + ret = cppc_get_perf_ctrs_sample(cpu, &fb_ctrs_t0, &fb_ctrs_t1); + if (ret) { + if (ret == -EFAULT) + /* Any of the associated CPPC regs is 0. */ + goto out_invalid_counters; + else + return 0; + } + + delivered_perf = cppc_perf_from_fbctrs(&fb_ctrs_t0, &fb_ctrs_t1); + if (!delivered_perf) + goto out_invalid_counters; + + return cppc_perf_to_khz(&cpu_data->perf_caps, delivered_perf); + +out_invalid_counters: + /* + * Feedback counters could be unchanged or 0 when a cpu enters a + * low-power idle state, e.g. clock-gated or power-gated. + * Use desired perf for reflecting frequency. Get the latest register + * value first as some platforms may update the actual delivered perf + * there; if failed, resort to the cached desired perf. + */ + if (cppc_get_desired_perf(cpu, &delivered_perf)) + delivered_perf = cpu_data->perf_ctrls.desired_perf; + + return cppc_perf_to_khz(&cpu_data->perf_caps, delivered_perf); +} + +static int cppc_cpufreq_set_boost(struct cpufreq_policy *policy, int state) +{ + struct cppc_cpudata *cpu_data = policy->driver_data; + struct cppc_perf_caps *caps = &cpu_data->perf_caps; + int ret; + + if (state) + policy->max = cppc_perf_to_khz(caps, caps->highest_perf); + else + policy->max = cppc_perf_to_khz(caps, caps->nominal_perf); + policy->cpuinfo.max_freq = policy->max; + + ret = freq_qos_update_request(policy->max_freq_req, policy->max); + if (ret < 0) + return ret; + + return 0; +} + +static ssize_t show_freqdomain_cpus(struct cpufreq_policy *policy, char *buf) +{ + struct cppc_cpudata *cpu_data = policy->driver_data; + + return cpufreq_show_cpus(cpu_data->shared_cpu_map, buf); +} + +static ssize_t show_auto_select(struct cpufreq_policy *policy, char *buf) +{ + bool val; + int ret; + + ret = cppc_get_auto_sel(policy->cpu, &val); + + /* show "<unsupported>" when this register is not supported by cpc */ + if (ret == -EOPNOTSUPP) + return sysfs_emit(buf, "<unsupported>\n"); + + if (ret) + return ret; + + return sysfs_emit(buf, "%d\n", val); +} + +static ssize_t store_auto_select(struct cpufreq_policy *policy, + const char *buf, size_t count) +{ + bool val; + int ret; + + ret = kstrtobool(buf, &val); + if (ret) + return ret; + + ret = cppc_set_auto_sel(policy->cpu, val); + if (ret) + return ret; + + return count; +} + +static ssize_t show_auto_act_window(struct cpufreq_policy *policy, char *buf) +{ + u64 val; + int ret; + + ret = cppc_get_auto_act_window(policy->cpu, &val); + + /* show "<unsupported>" when this register is not supported by cpc */ + if (ret == -EOPNOTSUPP) + return sysfs_emit(buf, "<unsupported>\n"); + + if (ret) + return ret; + + return sysfs_emit(buf, "%llu\n", val); +} + +static ssize_t store_auto_act_window(struct cpufreq_policy *policy, + const char *buf, size_t count) +{ + u64 usec; + int ret; + + ret = kstrtou64(buf, 0, &usec); + if (ret) + return ret; + + ret = cppc_set_auto_act_window(policy->cpu, usec); + if (ret) + return ret; + + return count; +} + +static ssize_t show_energy_performance_preference_val(struct cpufreq_policy *policy, char *buf) +{ + u64 val; + int ret; + + ret = cppc_get_epp_perf(policy->cpu, &val); + + /* show "<unsupported>" when this register is not supported by cpc */ + if (ret == -EOPNOTSUPP) + return sysfs_emit(buf, "<unsupported>\n"); + + if (ret) + return ret; + + return sysfs_emit(buf, "%llu\n", val); +} + +static ssize_t store_energy_performance_preference_val(struct cpufreq_policy *policy, + const char *buf, size_t count) +{ + u64 val; + int ret; + + ret = kstrtou64(buf, 0, &val); + if (ret) + return ret; + + ret = cppc_set_epp(policy->cpu, val); + if (ret) + return ret; + + return count; +} + +cpufreq_freq_attr_ro(freqdomain_cpus); +cpufreq_freq_attr_rw(auto_select); +cpufreq_freq_attr_rw(auto_act_window); +cpufreq_freq_attr_rw(energy_performance_preference_val); + +static struct freq_attr *cppc_cpufreq_attr[] = { + &freqdomain_cpus, + &auto_select, + &auto_act_window, + &energy_performance_preference_val, + NULL, +}; + +static struct cpufreq_driver cppc_cpufreq_driver = { + .flags = CPUFREQ_CONST_LOOPS | CPUFREQ_NEED_UPDATE_LIMITS, + .verify = cppc_verify_policy, + .target = cppc_cpufreq_set_target, + .get = cppc_cpufreq_get_rate, + .fast_switch = cppc_cpufreq_fast_switch, + .init = cppc_cpufreq_cpu_init, + .exit = cppc_cpufreq_cpu_exit, + .set_boost = cppc_cpufreq_set_boost, + .attr = cppc_cpufreq_attr, + .name = "cppc_cpufreq", +}; + +static int __init cppc_cpufreq_init(void) +{ + int ret; + + if (!acpi_cpc_valid()) + return -ENODEV; + + cppc_freq_invariance_init(); + populate_efficiency_class(); + + ret = cpufreq_register_driver(&cppc_cpufreq_driver); + if (ret) + cppc_freq_invariance_exit(); + + return ret; +} + +static void __exit cppc_cpufreq_exit(void) +{ + cpufreq_unregister_driver(&cppc_cpufreq_driver); + cppc_freq_invariance_exit(); +} + +module_exit(cppc_cpufreq_exit); +MODULE_AUTHOR("Ashwin Chaugule"); +MODULE_DESCRIPTION("CPUFreq driver based on the ACPI CPPC v5.0+ spec"); +MODULE_LICENSE("GPL"); + +late_initcall(cppc_cpufreq_init); + +static const struct acpi_device_id cppc_acpi_ids[] __used = { + {ACPI_PROCESSOR_DEVICE_HID, }, + {} +}; + +MODULE_DEVICE_TABLE(acpi, cppc_acpi_ids); diff --git a/drivers/cpufreq/cpufreq-cpu0.c b/drivers/cpufreq/cpufreq-cpu0.c deleted file mode 100644 index ad1fde277661..000000000000 --- a/drivers/cpufreq/cpufreq-cpu0.c +++ /dev/null @@ -1,304 +0,0 @@ -/* - * Copyright (C) 2012 Freescale Semiconductor, Inc. - * - * The OPP code in function cpu0_set_target() is reused from - * drivers/cpufreq/omap-cpufreq.c - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - */ - -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - -#include <linux/clk.h> -#include <linux/cpufreq.h> -#include <linux/err.h> -#include <linux/module.h> -#include <linux/of.h> -#include <linux/opp.h> -#include <linux/platform_device.h> -#include <linux/regulator/consumer.h> -#include <linux/slab.h> - -static unsigned int transition_latency; -static unsigned int voltage_tolerance; /* in percentage */ - -static struct device *cpu_dev; -static struct clk *cpu_clk; -static struct regulator *cpu_reg; -static struct cpufreq_frequency_table *freq_table; - -static int cpu0_verify_speed(struct cpufreq_policy *policy) -{ - return cpufreq_frequency_table_verify(policy, freq_table); -} - -static unsigned int cpu0_get_speed(unsigned int cpu) -{ - return clk_get_rate(cpu_clk) / 1000; -} - -static int cpu0_set_target(struct cpufreq_policy *policy, - unsigned int target_freq, unsigned int relation) -{ - struct cpufreq_freqs freqs; - struct opp *opp; - unsigned long volt = 0, volt_old = 0, tol = 0; - long freq_Hz, freq_exact; - unsigned int index; - int ret; - - ret = cpufreq_frequency_table_target(policy, freq_table, target_freq, - relation, &index); - if (ret) { - pr_err("failed to match target freqency %d: %d\n", - target_freq, ret); - return ret; - } - - freq_Hz = clk_round_rate(cpu_clk, freq_table[index].frequency * 1000); - if (freq_Hz < 0) - freq_Hz = freq_table[index].frequency * 1000; - freq_exact = freq_Hz; - freqs.new = freq_Hz / 1000; - freqs.old = clk_get_rate(cpu_clk) / 1000; - - if (freqs.old == freqs.new) - return 0; - - cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE); - - if (cpu_reg) { - rcu_read_lock(); - opp = opp_find_freq_ceil(cpu_dev, &freq_Hz); - if (IS_ERR(opp)) { - rcu_read_unlock(); - pr_err("failed to find OPP for %ld\n", freq_Hz); - freqs.new = freqs.old; - ret = PTR_ERR(opp); - goto post_notify; - } - volt = opp_get_voltage(opp); - rcu_read_unlock(); - tol = volt * voltage_tolerance / 100; - volt_old = regulator_get_voltage(cpu_reg); - } - - pr_debug("%u MHz, %ld mV --> %u MHz, %ld mV\n", - freqs.old / 1000, volt_old ? volt_old / 1000 : -1, - freqs.new / 1000, volt ? volt / 1000 : -1); - - /* scaling up? scale voltage before frequency */ - if (cpu_reg && freqs.new > freqs.old) { - ret = regulator_set_voltage_tol(cpu_reg, volt, tol); - if (ret) { - pr_err("failed to scale voltage up: %d\n", ret); - freqs.new = freqs.old; - goto post_notify; - } - } - - ret = clk_set_rate(cpu_clk, freq_exact); - if (ret) { - pr_err("failed to set clock rate: %d\n", ret); - if (cpu_reg) - regulator_set_voltage_tol(cpu_reg, volt_old, tol); - freqs.new = freqs.old; - goto post_notify; - } - - /* scaling down? scale voltage after frequency */ - if (cpu_reg && freqs.new < freqs.old) { - ret = regulator_set_voltage_tol(cpu_reg, volt, tol); - if (ret) { - pr_err("failed to scale voltage down: %d\n", ret); - clk_set_rate(cpu_clk, freqs.old * 1000); - freqs.new = freqs.old; - } - } - -post_notify: - cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); - - return ret; -} - -static int cpu0_cpufreq_init(struct cpufreq_policy *policy) -{ - int ret; - - ret = cpufreq_frequency_table_cpuinfo(policy, freq_table); - if (ret) { - pr_err("invalid frequency table: %d\n", ret); - return ret; - } - - policy->cpuinfo.transition_latency = transition_latency; - policy->cur = clk_get_rate(cpu_clk) / 1000; - - /* - * The driver only supports the SMP configuartion where all processors - * share the clock and voltage and clock. Use cpufreq affected_cpus - * interface to have all CPUs scaled together. - */ - cpumask_setall(policy->cpus); - - cpufreq_frequency_table_get_attr(freq_table, policy->cpu); - - return 0; -} - -static int cpu0_cpufreq_exit(struct cpufreq_policy *policy) -{ - cpufreq_frequency_table_put_attr(policy->cpu); - - return 0; -} - -static struct freq_attr *cpu0_cpufreq_attr[] = { - &cpufreq_freq_attr_scaling_available_freqs, - NULL, -}; - -static struct cpufreq_driver cpu0_cpufreq_driver = { - .flags = CPUFREQ_STICKY, - .verify = cpu0_verify_speed, - .target = cpu0_set_target, - .get = cpu0_get_speed, - .init = cpu0_cpufreq_init, - .exit = cpu0_cpufreq_exit, - .name = "generic_cpu0", - .attr = cpu0_cpufreq_attr, -}; - -static int cpu0_cpufreq_probe(struct platform_device *pdev) -{ - struct device_node *np, *parent; - int ret; - - parent = of_find_node_by_path("/cpus"); - if (!parent) { - pr_err("failed to find OF /cpus\n"); - return -ENOENT; - } - - for_each_child_of_node(parent, np) { - if (of_get_property(np, "operating-points", NULL)) - break; - } - - if (!np) { - pr_err("failed to find cpu0 node\n"); - ret = -ENOENT; - goto out_put_parent; - } - - cpu_dev = &pdev->dev; - cpu_dev->of_node = np; - - cpu_reg = devm_regulator_get(cpu_dev, "cpu0"); - if (IS_ERR(cpu_reg)) { - /* - * If cpu0 regulator supply node is present, but regulator is - * not yet registered, we should try defering probe. - */ - if (PTR_ERR(cpu_reg) == -EPROBE_DEFER) { - dev_err(cpu_dev, "cpu0 regulator not ready, retry\n"); - ret = -EPROBE_DEFER; - goto out_put_node; - } - pr_warn("failed to get cpu0 regulator: %ld\n", - PTR_ERR(cpu_reg)); - cpu_reg = NULL; - } - - cpu_clk = devm_clk_get(cpu_dev, NULL); - if (IS_ERR(cpu_clk)) { - ret = PTR_ERR(cpu_clk); - pr_err("failed to get cpu0 clock: %d\n", ret); - goto out_put_node; - } - - ret = of_init_opp_table(cpu_dev); - if (ret) { - pr_err("failed to init OPP table: %d\n", ret); - goto out_put_node; - } - - ret = opp_init_cpufreq_table(cpu_dev, &freq_table); - if (ret) { - pr_err("failed to init cpufreq table: %d\n", ret); - goto out_put_node; - } - - of_property_read_u32(np, "voltage-tolerance", &voltage_tolerance); - - if (of_property_read_u32(np, "clock-latency", &transition_latency)) - transition_latency = CPUFREQ_ETERNAL; - - if (cpu_reg) { - struct opp *opp; - unsigned long min_uV, max_uV; - int i; - - /* - * OPP is maintained in order of increasing frequency, and - * freq_table initialised from OPP is therefore sorted in the - * same order. - */ - for (i = 0; freq_table[i].frequency != CPUFREQ_TABLE_END; i++) - ; - rcu_read_lock(); - opp = opp_find_freq_exact(cpu_dev, - freq_table[0].frequency * 1000, true); - min_uV = opp_get_voltage(opp); - opp = opp_find_freq_exact(cpu_dev, - freq_table[i-1].frequency * 1000, true); - max_uV = opp_get_voltage(opp); - rcu_read_unlock(); - ret = regulator_set_voltage_time(cpu_reg, min_uV, max_uV); - if (ret > 0) - transition_latency += ret * 1000; - } - - ret = cpufreq_register_driver(&cpu0_cpufreq_driver); - if (ret) { - pr_err("failed register driver: %d\n", ret); - goto out_free_table; - } - - of_node_put(np); - of_node_put(parent); - return 0; - -out_free_table: - opp_free_cpufreq_table(cpu_dev, &freq_table); -out_put_node: - of_node_put(np); -out_put_parent: - of_node_put(parent); - return ret; -} - -static int cpu0_cpufreq_remove(struct platform_device *pdev) -{ - cpufreq_unregister_driver(&cpu0_cpufreq_driver); - opp_free_cpufreq_table(cpu_dev, &freq_table); - - return 0; -} - -static struct platform_driver cpu0_cpufreq_platdrv = { - .driver = { - .name = "cpufreq-cpu0", - .owner = THIS_MODULE, - }, - .probe = cpu0_cpufreq_probe, - .remove = cpu0_cpufreq_remove, -}; -module_platform_driver(cpu0_cpufreq_platdrv); - -MODULE_AUTHOR("Shawn Guo <shawn.guo@linaro.org>"); -MODULE_DESCRIPTION("Generic CPU0 cpufreq driver"); -MODULE_LICENSE("GPL"); diff --git a/drivers/cpufreq/cpufreq-dt-platdev.c b/drivers/cpufreq/cpufreq-dt-platdev.c new file mode 100644 index 000000000000..a1d11ecd1ac8 --- /dev/null +++ b/drivers/cpufreq/cpufreq-dt-platdev.c @@ -0,0 +1,238 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2016 Linaro. + * Viresh Kumar <viresh.kumar@linaro.org> + */ + +#include <linux/err.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/platform_device.h> + +#include "cpufreq-dt.h" + +/* + * Machines for which the cpufreq device is *always* created, mostly used for + * platforms using "operating-points" (V1) property. + */ +static const struct of_device_id allowlist[] __initconst = { + { .compatible = "allwinner,sun4i-a10", }, + { .compatible = "allwinner,sun5i-a10s", }, + { .compatible = "allwinner,sun5i-a13", }, + { .compatible = "allwinner,sun5i-r8", }, + { .compatible = "allwinner,sun6i-a31", }, + { .compatible = "allwinner,sun6i-a31s", }, + { .compatible = "allwinner,sun7i-a20", }, + { .compatible = "allwinner,sun8i-a23", }, + { .compatible = "allwinner,sun8i-a83t", }, + { .compatible = "allwinner,sun8i-h3", }, + + { .compatible = "apm,xgene-shadowcat", }, + + { .compatible = "arm,integrator-ap", }, + { .compatible = "arm,integrator-cp", }, + + { .compatible = "hisilicon,hi3660", }, + + { .compatible = "fsl,imx27", }, + { .compatible = "fsl,imx51", }, + { .compatible = "fsl,imx53", }, + + { .compatible = "marvell,berlin", }, + { .compatible = "marvell,pxa250", }, + { .compatible = "marvell,pxa270", }, + + { .compatible = "samsung,exynos3250", }, + { .compatible = "samsung,exynos4210", }, + { .compatible = "samsung,exynos5250", }, +#ifndef CONFIG_BL_SWITCHER + { .compatible = "samsung,exynos5800", }, +#endif + + { .compatible = "renesas,emev2", }, + { .compatible = "renesas,r7s72100", }, + { .compatible = "renesas,r8a73a4", }, + { .compatible = "renesas,r8a7740", }, + { .compatible = "renesas,r8a7742", }, + { .compatible = "renesas,r8a7743", }, + { .compatible = "renesas,r8a7744", }, + { .compatible = "renesas,r8a7745", }, + { .compatible = "renesas,r8a7778", }, + { .compatible = "renesas,r8a7779", }, + { .compatible = "renesas,r8a7790", }, + { .compatible = "renesas,r8a7791", }, + { .compatible = "renesas,r8a7792", }, + { .compatible = "renesas,r8a7793", }, + { .compatible = "renesas,r8a7794", }, + { .compatible = "renesas,sh73a0", }, + + { .compatible = "rockchip,rk2928", }, + { .compatible = "rockchip,rk3036", }, + { .compatible = "rockchip,rk3066a", }, + { .compatible = "rockchip,rk3066b", }, + { .compatible = "rockchip,rk3188", }, + { .compatible = "rockchip,rk3228", }, + { .compatible = "rockchip,rk3288", }, + { .compatible = "rockchip,rk3328", }, + { .compatible = "rockchip,rk3366", }, + { .compatible = "rockchip,rk3368", }, + { .compatible = "rockchip,rk3399", + .data = &(struct cpufreq_dt_platform_data) + { .have_governor_per_policy = true, }, + }, + + { .compatible = "st-ericsson,u8500", }, + { .compatible = "st-ericsson,u8540", }, + { .compatible = "st-ericsson,u9500", }, + { .compatible = "st-ericsson,u9540", }, + + { .compatible = "starfive,jh7110", }, + { .compatible = "starfive,jh7110s", }, + + { .compatible = "ti,omap2", }, + { .compatible = "ti,omap4", }, + { .compatible = "ti,omap5", }, + + { .compatible = "xlnx,zynq-7000", }, + { .compatible = "xlnx,zynqmp", }, + + { } +}; + +/* + * Machines for which the cpufreq device is *not* created, mostly used for + * platforms using "operating-points-v2" property. + */ +static const struct of_device_id blocklist[] __initconst = { + { .compatible = "airoha,an7583", }, + { .compatible = "airoha,en7581", }, + + { .compatible = "allwinner,sun50i-a100" }, + { .compatible = "allwinner,sun50i-h6", }, + { .compatible = "allwinner,sun50i-h616", }, + { .compatible = "allwinner,sun50i-h618", }, + { .compatible = "allwinner,sun50i-h700", }, + + { .compatible = "apple,arm-platform", }, + + { .compatible = "arm,vexpress", }, + + { .compatible = "calxeda,highbank", }, + { .compatible = "calxeda,ecx-2000", }, + + { .compatible = "fsl,imx7ulp", }, + { .compatible = "fsl,imx7d", }, + { .compatible = "fsl,imx7s", }, + { .compatible = "fsl,imx8mq", }, + { .compatible = "fsl,imx8mm", }, + { .compatible = "fsl,imx8mn", }, + { .compatible = "fsl,imx8mp", }, + + { .compatible = "marvell,armadaxp", }, + + { .compatible = "mediatek,mt2701", }, + { .compatible = "mediatek,mt2712", }, + { .compatible = "mediatek,mt7622", }, + { .compatible = "mediatek,mt7623", }, + { .compatible = "mediatek,mt8167", }, + { .compatible = "mediatek,mt817x", }, + { .compatible = "mediatek,mt8173", }, + { .compatible = "mediatek,mt8176", }, + { .compatible = "mediatek,mt8183", }, + { .compatible = "mediatek,mt8186", }, + { .compatible = "mediatek,mt8365", }, + { .compatible = "mediatek,mt8516", }, + + { .compatible = "nvidia,tegra20", }, + { .compatible = "nvidia,tegra30", }, + { .compatible = "nvidia,tegra114", }, + { .compatible = "nvidia,tegra124", }, + { .compatible = "nvidia,tegra210", }, + { .compatible = "nvidia,tegra234", }, + + { .compatible = "qcom,apq8096", }, + { .compatible = "qcom,msm8909", }, + { .compatible = "qcom,msm8996", }, + { .compatible = "qcom,msm8998", }, + { .compatible = "qcom,qcm2290", }, + { .compatible = "qcom,qcm6490", }, + { .compatible = "qcom,qcs404", }, + { .compatible = "qcom,qdu1000", }, + { .compatible = "qcom,sa8155p" }, + { .compatible = "qcom,sa8540p" }, + { .compatible = "qcom,sa8775p" }, + { .compatible = "qcom,sc7180", }, + { .compatible = "qcom,sc7280", }, + { .compatible = "qcom,sc8180x", }, + { .compatible = "qcom,sc8280xp", }, + { .compatible = "qcom,sdm670", }, + { .compatible = "qcom,sdm845", }, + { .compatible = "qcom,sdx75", }, + { .compatible = "qcom,sm6115", }, + { .compatible = "qcom,sm6350", }, + { .compatible = "qcom,sm6375", }, + { .compatible = "qcom,sm7225", }, + { .compatible = "qcom,sm7325", }, + { .compatible = "qcom,sm8150", }, + { .compatible = "qcom,sm8250", }, + { .compatible = "qcom,sm8350", }, + { .compatible = "qcom,sm8450", }, + { .compatible = "qcom,sm8550", }, + { .compatible = "qcom,sm8650", }, + + { .compatible = "st,stih407", }, + { .compatible = "st,stih410", }, + { .compatible = "st,stih418", }, + + { .compatible = "ti,am33xx", }, + { .compatible = "ti,am43", }, + { .compatible = "ti,dra7", }, + { .compatible = "ti,omap3", }, + { .compatible = "ti,am625", }, + { .compatible = "ti,am62a7", }, + { .compatible = "ti,am62d2", }, + { .compatible = "ti,am62p5", }, + + { .compatible = "qcom,ipq5332", }, + { .compatible = "qcom,ipq5424", }, + { .compatible = "qcom,ipq6018", }, + { .compatible = "qcom,ipq8064", }, + { .compatible = "qcom,ipq8074", }, + { .compatible = "qcom,ipq9574", }, + { .compatible = "qcom,apq8064", }, + { .compatible = "qcom,msm8974", }, + { .compatible = "qcom,msm8960", }, + + { } +}; + +static bool __init cpu0_node_has_opp_v2_prop(void) +{ + struct device_node *np __free(device_node) = of_cpu_device_node_get(0); + bool ret = false; + + if (of_property_present(np, "operating-points-v2")) + ret = true; + + return ret; +} + +static int __init cpufreq_dt_platdev_init(void) +{ + const void *data; + + data = of_machine_get_match_data(allowlist); + if (data) + goto create_pdev; + + if (cpu0_node_has_opp_v2_prop() && !of_machine_device_match(blocklist)) + goto create_pdev; + + return -ENODEV; + +create_pdev: + return PTR_ERR_OR_ZERO(platform_device_register_data(NULL, "cpufreq-dt", + -1, data, + sizeof(struct cpufreq_dt_platform_data))); +} +core_initcall(cpufreq_dt_platdev_init); diff --git a/drivers/cpufreq/cpufreq-dt.c b/drivers/cpufreq/cpufreq-dt.c new file mode 100644 index 000000000000..7d5079fd1688 --- /dev/null +++ b/drivers/cpufreq/cpufreq-dt.c @@ -0,0 +1,347 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2012 Freescale Semiconductor, Inc. + * + * Copyright (C) 2014 Linaro. + * Viresh Kumar <viresh.kumar@linaro.org> + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/clk.h> +#include <linux/cpu.h> +#include <linux/cpufreq.h> +#include <linux/cpumask.h> +#include <linux/err.h> +#include <linux/list.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/pm_opp.h> +#include <linux/platform_device.h> +#include <linux/regulator/consumer.h> +#include <linux/slab.h> +#include <linux/thermal.h> + +#include "cpufreq-dt.h" + +struct private_data { + struct list_head node; + + cpumask_var_t cpus; + struct device *cpu_dev; + struct cpufreq_frequency_table *freq_table; + bool have_static_opps; + int opp_token; +}; + +static LIST_HEAD(priv_list); + +static struct private_data *cpufreq_dt_find_data(int cpu) +{ + struct private_data *priv; + + list_for_each_entry(priv, &priv_list, node) { + if (cpumask_test_cpu(cpu, priv->cpus)) + return priv; + } + + return NULL; +} + +static int set_target(struct cpufreq_policy *policy, unsigned int index) +{ + struct private_data *priv = policy->driver_data; + unsigned long freq = policy->freq_table[index].frequency; + + return dev_pm_opp_set_rate(priv->cpu_dev, freq * 1000); +} + +/* + * An earlier version of opp-v1 bindings used to name the regulator + * "cpu0-supply", we still need to handle that for backwards compatibility. + */ +static const char *find_supply_name(struct device *dev) +{ + struct device_node *np __free(device_node) = of_node_get(dev->of_node); + int cpu = dev->id; + + /* This must be valid for sure */ + if (WARN_ON(!np)) + return NULL; + + /* Try "cpu0" for older DTs */ + if (!cpu && of_property_present(np, "cpu0-supply")) + return "cpu0"; + + if (of_property_present(np, "cpu-supply")) + return "cpu"; + + dev_dbg(dev, "no regulator for cpu%d\n", cpu); + return NULL; +} + +static int cpufreq_init(struct cpufreq_policy *policy) +{ + struct private_data *priv; + struct device *cpu_dev; + struct clk *cpu_clk; + unsigned int transition_latency; + int ret; + + priv = cpufreq_dt_find_data(policy->cpu); + if (!priv) { + pr_err("failed to find data for cpu%d\n", policy->cpu); + return -ENODEV; + } + cpu_dev = priv->cpu_dev; + + cpu_clk = clk_get(cpu_dev, NULL); + if (IS_ERR(cpu_clk)) { + ret = PTR_ERR(cpu_clk); + dev_err(cpu_dev, "%s: failed to get clk: %d\n", __func__, ret); + return ret; + } + + transition_latency = dev_pm_opp_get_max_transition_latency(cpu_dev); + if (!transition_latency) + transition_latency = CPUFREQ_DEFAULT_TRANSITION_LATENCY_NS; + + cpumask_copy(policy->cpus, priv->cpus); + policy->driver_data = priv; + policy->clk = cpu_clk; + policy->freq_table = priv->freq_table; + policy->suspend_freq = dev_pm_opp_get_suspend_opp_freq(cpu_dev) / 1000; + policy->cpuinfo.transition_latency = transition_latency; + policy->dvfs_possible_from_any_cpu = true; + + return 0; +} + +static int cpufreq_online(struct cpufreq_policy *policy) +{ + /* We did light-weight tear down earlier, nothing to do here */ + return 0; +} + +static int cpufreq_offline(struct cpufreq_policy *policy) +{ + /* + * Preserve policy->driver_data and don't free resources on light-weight + * tear down. + */ + return 0; +} + +static void cpufreq_exit(struct cpufreq_policy *policy) +{ + clk_put(policy->clk); +} + +static struct cpufreq_driver dt_cpufreq_driver = { + .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK | + CPUFREQ_IS_COOLING_DEV, + .verify = cpufreq_generic_frequency_table_verify, + .target_index = set_target, + .get = cpufreq_generic_get, + .init = cpufreq_init, + .exit = cpufreq_exit, + .online = cpufreq_online, + .offline = cpufreq_offline, + .register_em = cpufreq_register_em_with_opp, + .name = "cpufreq-dt", + .set_boost = cpufreq_boost_set_sw, + .suspend = cpufreq_generic_suspend, +}; + +static int dt_cpufreq_early_init(struct device *dev, int cpu) +{ + struct private_data *priv; + struct device *cpu_dev; + bool fallback = false; + const char *reg_name[] = { NULL, NULL }; + int ret; + + /* Check if this CPU is already covered by some other policy */ + if (cpufreq_dt_find_data(cpu)) + return 0; + + cpu_dev = get_cpu_device(cpu); + if (!cpu_dev) + return -EPROBE_DEFER; + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + if (!zalloc_cpumask_var(&priv->cpus, GFP_KERNEL)) + return -ENOMEM; + + cpumask_set_cpu(cpu, priv->cpus); + priv->cpu_dev = cpu_dev; + + /* + * OPP layer will be taking care of regulators now, but it needs to know + * the name of the regulator first. + */ + reg_name[0] = find_supply_name(cpu_dev); + if (reg_name[0]) { + priv->opp_token = dev_pm_opp_set_regulators(cpu_dev, reg_name); + if (priv->opp_token < 0) { + ret = dev_err_probe(cpu_dev, priv->opp_token, + "failed to set regulators\n"); + goto free_cpumask; + } + } + + /* Get OPP-sharing information from "operating-points-v2" bindings */ + ret = dev_pm_opp_of_get_sharing_cpus(cpu_dev, priv->cpus); + if (ret) { + if (ret != -ENOENT) + goto out; + + /* + * operating-points-v2 not supported, fallback to all CPUs share + * OPP for backward compatibility if the platform hasn't set + * sharing CPUs. + */ + if (dev_pm_opp_get_sharing_cpus(cpu_dev, priv->cpus)) + fallback = true; + } + + /* + * Initialize OPP tables for all priv->cpus. They will be shared by + * all CPUs which have marked their CPUs shared with OPP bindings. + * + * For platforms not using operating-points-v2 bindings, we do this + * before updating priv->cpus. Otherwise, we will end up creating + * duplicate OPPs for the CPUs. + * + * OPPs might be populated at runtime, don't fail for error here unless + * it is -EPROBE_DEFER. + */ + ret = dev_pm_opp_of_cpumask_add_table(priv->cpus); + if (!ret) { + priv->have_static_opps = true; + } else if (ret == -EPROBE_DEFER) { + goto out; + } + + /* + * The OPP table must be initialized, statically or dynamically, by this + * point. + */ + ret = dev_pm_opp_get_opp_count(cpu_dev); + if (ret <= 0) { + dev_err(cpu_dev, "OPP table can't be empty\n"); + ret = -ENODEV; + goto out; + } + + if (fallback) { + cpumask_setall(priv->cpus); + ret = dev_pm_opp_set_sharing_cpus(cpu_dev, priv->cpus); + if (ret) + dev_err(cpu_dev, "%s: failed to mark OPPs as shared: %d\n", + __func__, ret); + } + + ret = dev_pm_opp_init_cpufreq_table(cpu_dev, &priv->freq_table); + if (ret) { + dev_err(cpu_dev, "failed to init cpufreq table: %d\n", ret); + goto out; + } + + list_add(&priv->node, &priv_list); + return 0; + +out: + if (priv->have_static_opps) + dev_pm_opp_of_cpumask_remove_table(priv->cpus); + dev_pm_opp_put_regulators(priv->opp_token); +free_cpumask: + free_cpumask_var(priv->cpus); + return ret; +} + +static void dt_cpufreq_release(void) +{ + struct private_data *priv, *tmp; + + list_for_each_entry_safe(priv, tmp, &priv_list, node) { + dev_pm_opp_free_cpufreq_table(priv->cpu_dev, &priv->freq_table); + if (priv->have_static_opps) + dev_pm_opp_of_cpumask_remove_table(priv->cpus); + dev_pm_opp_put_regulators(priv->opp_token); + free_cpumask_var(priv->cpus); + list_del(&priv->node); + } +} + +static int dt_cpufreq_probe(struct platform_device *pdev) +{ + struct cpufreq_dt_platform_data *data = dev_get_platdata(&pdev->dev); + int ret, cpu; + + /* Request resources early so we can return in case of -EPROBE_DEFER */ + for_each_present_cpu(cpu) { + ret = dt_cpufreq_early_init(&pdev->dev, cpu); + if (ret) + goto err; + } + + if (data) { + if (data->have_governor_per_policy) + dt_cpufreq_driver.flags |= CPUFREQ_HAVE_GOVERNOR_PER_POLICY; + + dt_cpufreq_driver.resume = data->resume; + if (data->suspend) + dt_cpufreq_driver.suspend = data->suspend; + if (data->get_intermediate) { + dt_cpufreq_driver.target_intermediate = data->target_intermediate; + dt_cpufreq_driver.get_intermediate = data->get_intermediate; + } + } + + ret = cpufreq_register_driver(&dt_cpufreq_driver); + if (ret) { + dev_err(&pdev->dev, "failed register driver: %d\n", ret); + goto err; + } + + return 0; +err: + dt_cpufreq_release(); + return ret; +} + +static void dt_cpufreq_remove(struct platform_device *pdev) +{ + cpufreq_unregister_driver(&dt_cpufreq_driver); + dt_cpufreq_release(); +} + +static struct platform_driver dt_cpufreq_platdrv = { + .driver = { + .name = "cpufreq-dt", + }, + .probe = dt_cpufreq_probe, + .remove = dt_cpufreq_remove, +}; +module_platform_driver(dt_cpufreq_platdrv); + +struct platform_device *cpufreq_dt_pdev_register(struct device *dev) +{ + struct platform_device_info cpufreq_dt_devinfo = {}; + + cpufreq_dt_devinfo.name = "cpufreq-dt"; + cpufreq_dt_devinfo.parent = dev; + + return platform_device_register_full(&cpufreq_dt_devinfo); +} +EXPORT_SYMBOL_GPL(cpufreq_dt_pdev_register); + +MODULE_ALIAS("platform:cpufreq-dt"); +MODULE_AUTHOR("Viresh Kumar <viresh.kumar@linaro.org>"); +MODULE_AUTHOR("Shawn Guo <shawn.guo@linaro.org>"); +MODULE_DESCRIPTION("Generic cpufreq driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/cpufreq/cpufreq-dt.h b/drivers/cpufreq/cpufreq-dt.h new file mode 100644 index 000000000000..fc1889aeb4f1 --- /dev/null +++ b/drivers/cpufreq/cpufreq-dt.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2016 Linaro + * Viresh Kumar <viresh.kumar@linaro.org> + */ + +#ifndef __CPUFREQ_DT_H__ +#define __CPUFREQ_DT_H__ + +#include <linux/types.h> + +struct cpufreq_policy; + +struct cpufreq_dt_platform_data { + bool have_governor_per_policy; + + unsigned int (*get_intermediate)(struct cpufreq_policy *policy, + unsigned int index); + int (*target_intermediate)(struct cpufreq_policy *policy, + unsigned int index); + int (*suspend)(struct cpufreq_policy *policy); + int (*resume)(struct cpufreq_policy *policy); +}; + +struct platform_device *cpufreq_dt_pdev_register(struct device *dev); + +#endif /* __CPUFREQ_DT_H__ */ diff --git a/drivers/cpufreq/cpufreq-nforce2.c b/drivers/cpufreq/cpufreq-nforce2.c index af1542d41440..fbbbe501cf2d 100644 --- a/drivers/cpufreq/cpufreq-nforce2.c +++ b/drivers/cpufreq/cpufreq-nforce2.c @@ -1,12 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * (C) 2004-2006 Sebastian Witt <se.witt@gmx.net> * - * Licensed under the terms of the GNU GPL License version 2. * Based upon reverse engineered information * * BIG FAT DISCLAIMER: Work in progress code. Possibly *dangerous* */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + #include <linux/kernel.h> #include <linux/module.h> #include <linux/moduleparam.h> @@ -56,8 +58,6 @@ MODULE_PARM_DESC(fid, "CPU multiplier to use (11.5 = 115)"); MODULE_PARM_DESC(min_fsb, "Minimum FSB to use, if not defined: current FSB - 50"); -#define PFX "cpufreq-nforce2: " - /** * nforce2_calc_fsb - calculate FSB * @pll: PLL value @@ -123,8 +123,6 @@ static void nforce2_write_pll(int pll) /* Now write the value in all 64 registers */ for (temp = 0; temp <= 0x3f; temp++) pci_write_config_dword(nforce2_dev, NFORCE2_PLLREG, pll); - - return; } /** @@ -147,6 +145,8 @@ static unsigned int nforce2_fsb_read(int bootfsb) pci_read_config_dword(nforce2_sub5, NFORCE2_BOOTFSB, &fsb); fsb /= 1000000; + pci_dev_put(nforce2_sub5); + /* Check if PLL register is already set */ pci_read_config_byte(nforce2_dev, NFORCE2_PLLENABLE, (u8 *)&temp); @@ -174,13 +174,13 @@ static int nforce2_set_fsb(unsigned int fsb) int pll = 0; if ((fsb > max_fsb) || (fsb < NFORCE2_MIN_FSB)) { - printk(KERN_ERR PFX "FSB %d is out of range!\n", fsb); + pr_err("FSB %d is out of range!\n", fsb); return -EINVAL; } tfsb = nforce2_fsb_read(0); if (!tfsb) { - printk(KERN_ERR PFX "Error while reading the FSB\n"); + pr_err("Error while reading the FSB\n"); return -EINVAL; } @@ -270,14 +270,13 @@ static int nforce2_target(struct cpufreq_policy *policy, pr_debug("Old CPU frequency %d kHz, new %d kHz\n", freqs.old, freqs.new); - cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE); + cpufreq_freq_transition_begin(policy, &freqs); /* Disable IRQs */ /* local_irq_save(flags); */ if (nforce2_set_fsb(target_fsb) < 0) - printk(KERN_ERR PFX "Changing FSB to %d failed\n", - target_fsb); + pr_err("Changing FSB to %d failed\n", target_fsb); else pr_debug("Changed FSB successfully to %d\n", target_fsb); @@ -285,7 +284,7 @@ static int nforce2_target(struct cpufreq_policy *policy, /* Enable IRQs */ /* local_irq_restore(flags); */ - cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); + cpufreq_freq_transition_end(policy, &freqs, 0); return 0; } @@ -294,7 +293,7 @@ static int nforce2_target(struct cpufreq_policy *policy, * nforce2_verify - verifies a new CPUFreq policy * @policy: new policy */ -static int nforce2_verify(struct cpufreq_policy *policy) +static int nforce2_verify(struct cpufreq_policy_data *policy) { unsigned int fsb_pol_max; @@ -303,9 +302,7 @@ static int nforce2_verify(struct cpufreq_policy *policy) if (policy->min < (fsb_pol_max * fid * 100)) policy->max = (fsb_pol_max + 1) * fid * 100; - cpufreq_verify_within_limits(policy, - policy->cpuinfo.min_freq, - policy->cpuinfo.max_freq); + cpufreq_verify_within_cpu_limits(policy); return 0; } @@ -327,8 +324,7 @@ static int nforce2_cpu_init(struct cpufreq_policy *policy) /* FIX: Get FID from CPU */ if (!fid) { if (!cpu_khz) { - printk(KERN_WARNING PFX - "cpu_khz not set, can't calculate multiplier!\n"); + pr_warn("cpu_khz not set, can't calculate multiplier!\n"); return -ENODEV; } @@ -343,8 +339,8 @@ static int nforce2_cpu_init(struct cpufreq_policy *policy) } } - printk(KERN_INFO PFX "FSB currently at %i MHz, FID %d.%d\n", fsb, - fid / 10, fid % 10); + pr_info("FSB currently at %i MHz, FID %d.%d\n", + fsb, fid / 10, fid % 10); /* Set maximum FSB to FSB at boot time */ max_fsb = nforce2_fsb_read(1); @@ -361,29 +357,21 @@ static int nforce2_cpu_init(struct cpufreq_policy *policy) /* cpuinfo and default policy values */ policy->min = policy->cpuinfo.min_freq = min_fsb * fid * 100; policy->max = policy->cpuinfo.max_freq = max_fsb * fid * 100; - policy->cpuinfo.transition_latency = CPUFREQ_ETERNAL; - policy->cur = nforce2_get(policy->cpu); return 0; } -static int nforce2_cpu_exit(struct cpufreq_policy *policy) -{ - return 0; -} - static struct cpufreq_driver nforce2_driver = { .name = "nforce2", + .flags = CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING, .verify = nforce2_verify, .target = nforce2_target, .get = nforce2_get, .init = nforce2_cpu_init, - .exit = nforce2_cpu_exit, - .owner = THIS_MODULE, }; #ifdef MODULE -static DEFINE_PCI_DEVICE_TABLE(nforce2_ids) = { +static const struct pci_device_id nforce2_ids[] = { { PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE2 }, {} }; @@ -405,11 +393,9 @@ static int nforce2_detect_chipset(void) if (nforce2_dev == NULL) return -ENODEV; - printk(KERN_INFO PFX "Detected nForce2 chipset revision %X\n", - nforce2_dev->revision); - printk(KERN_INFO PFX - "FSB changing is maybe unstable and can lead to " - "crashes and data loss.\n"); + pr_info("Detected nForce2 chipset revision %X\n", + nforce2_dev->revision); + pr_info("FSB changing is maybe unstable and can lead to crashes and data loss\n"); return 0; } @@ -418,7 +404,7 @@ static int nforce2_detect_chipset(void) * nforce2_init - initializes the nForce2 CPUFreq driver * * Initializes the nForce2 FSB support. Returns -ENODEV on unsupported - * devices, -EINVAL on problems during initiatization, and zero on + * devices, -EINVAL on problems during initialization, and zero on * success. */ static int __init nforce2_init(void) @@ -427,7 +413,7 @@ static int __init nforce2_init(void) /* detect chipset */ if (nforce2_detect_chipset()) { - printk(KERN_INFO PFX "No nForce2 chipset.\n"); + pr_info("No nForce2 chipset\n"); return -ENODEV; } @@ -442,8 +428,8 @@ static int __init nforce2_init(void) static void __exit nforce2_exit(void) { cpufreq_unregister_driver(&nforce2_driver); + pci_dev_put(nforce2_dev); } module_init(nforce2_init); module_exit(nforce2_exit); - diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 0937b8d6c2a4..4472bb1ec83c 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * linux/drivers/cpufreq/cpufreq.c * @@ -9,35 +10,48 @@ * Added handling for CPU hotplug * Feb 2006 - Jacob Shin <jacob.shin@amd.com> * Fix handling for CPU hotplug -- affected CPUs - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -#include <asm/cputime.h> -#include <linux/kernel.h> -#include <linux/kernel_stat.h> -#include <linux/module.h> -#include <linux/init.h> -#include <linux/notifier.h> +#include <linux/cpu.h> #include <linux/cpufreq.h> +#include <linux/cpu_cooling.h> #include <linux/delay.h> -#include <linux/interrupt.h> -#include <linux/spinlock.h> -#include <linux/tick.h> #include <linux/device.h> -#include <linux/slab.h> -#include <linux/cpu.h> -#include <linux/completion.h> +#include <linux/init.h> +#include <linux/kernel_stat.h> +#include <linux/module.h> #include <linux/mutex.h> +#include <linux/pm_qos.h> +#include <linux/slab.h> +#include <linux/string_choices.h> +#include <linux/suspend.h> #include <linux/syscore_ops.h> - +#include <linux/tick.h> +#include <linux/units.h> #include <trace/events/power.h> -/** +static LIST_HEAD(cpufreq_policy_list); + +/* Macros to iterate over CPU policies */ +#define for_each_suitable_policy(__policy, __active) \ + list_for_each_entry(__policy, &cpufreq_policy_list, policy_list) \ + if ((__active) == !policy_is_inactive(__policy)) + +#define for_each_active_policy(__policy) \ + for_each_suitable_policy(__policy, true) +#define for_each_inactive_policy(__policy) \ + for_each_suitable_policy(__policy, false) + +/* Iterate over governors */ +static LIST_HEAD(cpufreq_governor_list); +#define for_each_governor(__governor) \ + list_for_each_entry(__governor, &cpufreq_governor_list, governor_list) + +static char default_governor[CPUFREQ_NAME_LEN]; + +/* * The "cpufreq driver" - the arch- or hardware-dependent low * level driver of CPUFreq support, and its spinlock. This lock * also protects the cpufreq_cpu_data array. @@ -45,64 +59,38 @@ static struct cpufreq_driver *cpufreq_driver; static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data); static DEFINE_RWLOCK(cpufreq_driver_lock); -static DEFINE_MUTEX(cpufreq_governor_lock); -#ifdef CONFIG_HOTPLUG_CPU -/* This one keeps track of the previously set governor of a removed CPU */ -static DEFINE_PER_CPU(char[CPUFREQ_NAME_LEN], cpufreq_cpu_governor); -#endif - -/* - * cpu_policy_rwsem is a per CPU reader-writer semaphore designed to cure - * all cpufreq/hotplug/workqueue/etc related lock issues. - * - * The rules for this semaphore: - * - Any routine that wants to read from the policy structure will - * do a down_read on this semaphore. - * - Any routine that will write to the policy structure and/or may take away - * the policy altogether (eg. CPU hotplug), will hold this lock in write - * mode before doing so. - * - * Additional rules: - * - Governor routines that can be called in cpufreq hotplug path should not - * take this sem as top level hotplug notifier handler takes this. - * - Lock should not be held across - * __cpufreq_governor(data, CPUFREQ_GOV_STOP); - */ -static DEFINE_PER_CPU(int, cpufreq_policy_cpu); -static DEFINE_PER_CPU(struct rw_semaphore, cpu_policy_rwsem); - -#define lock_policy_rwsem(mode, cpu) \ -static int lock_policy_rwsem_##mode(int cpu) \ -{ \ - int policy_cpu = per_cpu(cpufreq_policy_cpu, cpu); \ - BUG_ON(policy_cpu == -1); \ - down_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu)); \ - \ - return 0; \ +static DEFINE_STATIC_KEY_FALSE(cpufreq_freq_invariance); +bool cpufreq_supports_freq_invariance(void) +{ + return static_branch_likely(&cpufreq_freq_invariance); } -lock_policy_rwsem(read, cpu); -lock_policy_rwsem(write, cpu); +/* Flag to suspend/resume CPUFreq governors */ +static bool cpufreq_suspended; -#define unlock_policy_rwsem(mode, cpu) \ -static void unlock_policy_rwsem_##mode(int cpu) \ -{ \ - int policy_cpu = per_cpu(cpufreq_policy_cpu, cpu); \ - BUG_ON(policy_cpu == -1); \ - up_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu)); \ +static inline bool has_target(void) +{ + return cpufreq_driver->target_index || cpufreq_driver->target; } -unlock_policy_rwsem(read, cpu); -unlock_policy_rwsem(write, cpu); +bool has_target_index(void) +{ + return !!cpufreq_driver->target_index; +} /* internal prototypes */ -static int __cpufreq_governor(struct cpufreq_policy *policy, - unsigned int event); -static unsigned int __cpufreq_get(unsigned int cpu); -static void handle_update(struct work_struct *work); +static unsigned int __cpufreq_get(struct cpufreq_policy *policy); +static int cpufreq_init_governor(struct cpufreq_policy *policy); +static void cpufreq_exit_governor(struct cpufreq_policy *policy); +static void cpufreq_governor_limits(struct cpufreq_policy *policy); +static int cpufreq_set_policy(struct cpufreq_policy *policy, + struct cpufreq_governor *new_gov, + unsigned int new_pol); +static bool cpufreq_boost_supported(void); +static int cpufreq_boost_trigger_state(int state); -/** +/* * Two notifier lists: the "policy" list is involved in the * validation process for a new CPU frequency policy; the * "transition" list for kernel code that needs to handle @@ -110,16 +98,7 @@ static void handle_update(struct work_struct *work); * The mutex locks both lists. */ static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list); -static struct srcu_notifier_head cpufreq_transition_notifier_list; - -static bool init_cpufreq_transition_notifier_list_called; -static int __init init_cpufreq_transition_notifier_list(void) -{ - srcu_init_notifier_head(&cpufreq_transition_notifier_list); - init_cpufreq_transition_notifier_list_called = true; - return 0; -} -pure_initcall(init_cpufreq_transition_notifier_list); +SRCU_NOTIFIER_HEAD_STATIC(cpufreq_transition_notifier_list); static int off __read_mostly; static int cpufreq_disabled(void) @@ -130,15 +109,18 @@ void disable_cpufreq(void) { off = 1; } -static LIST_HEAD(cpufreq_governor_list); +EXPORT_SYMBOL_GPL(disable_cpufreq); + static DEFINE_MUTEX(cpufreq_governor_mutex); bool have_governor_per_policy(void) { - return cpufreq_driver->have_governor_per_policy; + return !!(cpufreq_driver->flags & CPUFREQ_HAVE_GOVERNOR_PER_POLICY); } EXPORT_SYMBOL_GPL(have_governor_per_policy); +static struct kobject *cpufreq_global_kobject; + struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy) { if (have_governor_per_policy()) @@ -150,24 +132,27 @@ EXPORT_SYMBOL_GPL(get_governor_parent_kobj); static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall) { - u64 idle_time; + struct kernel_cpustat kcpustat; u64 cur_wall_time; + u64 idle_time; u64 busy_time; - cur_wall_time = jiffies64_to_cputime64(get_jiffies_64()); + cur_wall_time = jiffies64_to_nsecs(get_jiffies_64()); - busy_time = kcpustat_cpu(cpu).cpustat[CPUTIME_USER]; - busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM]; - busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ]; - busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ]; - busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL]; - busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE]; + kcpustat_cpu_fetch(&kcpustat, cpu); + + busy_time = kcpustat.cpustat[CPUTIME_USER]; + busy_time += kcpustat.cpustat[CPUTIME_SYSTEM]; + busy_time += kcpustat.cpustat[CPUTIME_IRQ]; + busy_time += kcpustat.cpustat[CPUTIME_SOFTIRQ]; + busy_time += kcpustat.cpustat[CPUTIME_STEAL]; + busy_time += kcpustat.cpustat[CPUTIME_NICE]; idle_time = cur_wall_time - busy_time; if (wall) - *wall = cputime_to_usecs(cur_wall_time); + *wall = div_u64(cur_wall_time, NSEC_PER_USEC); - return cputime_to_usecs(idle_time); + return div_u64(idle_time, NSEC_PER_USEC); } u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy) @@ -183,261 +168,530 @@ u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy) } EXPORT_SYMBOL_GPL(get_cpu_idle_time); -static struct cpufreq_policy *__cpufreq_cpu_get(unsigned int cpu, bool sysfs) +/* + * This is a generic cpufreq init() routine which can be used by cpufreq + * drivers of SMP systems. It will do following: + * - validate & show freq table passed + * - set policies transition latency + * - policy->cpus with all possible CPUs + */ +void cpufreq_generic_init(struct cpufreq_policy *policy, + struct cpufreq_frequency_table *table, + unsigned int transition_latency) { - struct cpufreq_policy *data; - unsigned long flags; + policy->freq_table = table; + policy->cpuinfo.transition_latency = transition_latency; - if (cpu >= nr_cpu_ids) - goto err_out; - - /* get the cpufreq driver */ - read_lock_irqsave(&cpufreq_driver_lock, flags); - - if (!cpufreq_driver) - goto err_out_unlock; - - if (!try_module_get(cpufreq_driver->owner)) - goto err_out_unlock; + /* + * The driver only supports the SMP configuration where all processors + * share the clock and voltage and clock. + */ + cpumask_setall(policy->cpus); +} +EXPORT_SYMBOL_GPL(cpufreq_generic_init); - /* get the CPU */ - data = per_cpu(cpufreq_cpu_data, cpu); +struct cpufreq_policy *cpufreq_cpu_get_raw(unsigned int cpu) +{ + struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu); - if (!data) - goto err_out_put_module; + return policy && cpumask_test_cpu(cpu, policy->cpus) ? policy : NULL; +} +EXPORT_SYMBOL_GPL(cpufreq_cpu_get_raw); - if (!sysfs && !kobject_get(&data->kobj)) - goto err_out_put_module; +unsigned int cpufreq_generic_get(unsigned int cpu) +{ + struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); - read_unlock_irqrestore(&cpufreq_driver_lock, flags); - return data; + if (!policy || IS_ERR(policy->clk)) { + pr_err("%s: No %s associated to cpu: %d\n", + __func__, policy ? "clk" : "policy", cpu); + return 0; + } -err_out_put_module: - module_put(cpufreq_driver->owner); -err_out_unlock: - read_unlock_irqrestore(&cpufreq_driver_lock, flags); -err_out: - return NULL; + return clk_get_rate(policy->clk) / 1000; } +EXPORT_SYMBOL_GPL(cpufreq_generic_get); +/** + * cpufreq_cpu_get - Return policy for a CPU and mark it as busy. + * @cpu: CPU to find the policy for. + * + * Call cpufreq_cpu_get_raw() to obtain a cpufreq policy for @cpu and increment + * the kobject reference counter of that policy. Return a valid policy on + * success or NULL on failure. + * + * The policy returned by this function has to be released with the help of + * cpufreq_cpu_put() to balance its kobject reference counter properly. + */ struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu) { - if (cpufreq_disabled()) - return NULL; + struct cpufreq_policy *policy = NULL; + unsigned long flags; - return __cpufreq_cpu_get(cpu, false); -} -EXPORT_SYMBOL_GPL(cpufreq_cpu_get); + if (WARN_ON(cpu >= nr_cpu_ids)) + return NULL; -static struct cpufreq_policy *cpufreq_cpu_get_sysfs(unsigned int cpu) -{ - return __cpufreq_cpu_get(cpu, true); -} + /* get the cpufreq driver */ + read_lock_irqsave(&cpufreq_driver_lock, flags); -static void __cpufreq_cpu_put(struct cpufreq_policy *data, bool sysfs) -{ - if (!sysfs) - kobject_put(&data->kobj); - module_put(cpufreq_driver->owner); -} + if (cpufreq_driver) { + /* get the CPU */ + policy = cpufreq_cpu_get_raw(cpu); + if (policy) + kobject_get(&policy->kobj); + } -void cpufreq_cpu_put(struct cpufreq_policy *data) -{ - if (cpufreq_disabled()) - return; + read_unlock_irqrestore(&cpufreq_driver_lock, flags); - __cpufreq_cpu_put(data, false); + return policy; } -EXPORT_SYMBOL_GPL(cpufreq_cpu_put); +EXPORT_SYMBOL_GPL(cpufreq_cpu_get); -static void cpufreq_cpu_put_sysfs(struct cpufreq_policy *data) +/** + * cpufreq_cpu_put - Decrement kobject usage counter for cpufreq policy. + * @policy: cpufreq policy returned by cpufreq_cpu_get(). + */ +void cpufreq_cpu_put(struct cpufreq_policy *policy) { - __cpufreq_cpu_put(data, true); + kobject_put(&policy->kobj); } +EXPORT_SYMBOL_GPL(cpufreq_cpu_put); /********************************************************************* * EXTERNALLY AFFECTING FREQUENCY CHANGES * *********************************************************************/ /** - * adjust_jiffies - adjust the system "loops_per_jiffy" + * adjust_jiffies - Adjust the system "loops_per_jiffy". + * @val: CPUFREQ_PRECHANGE or CPUFREQ_POSTCHANGE. + * @ci: Frequency change information. * * This function alters the system "loops_per_jiffy" for the clock * speed change. Note that loops_per_jiffy cannot be updated on SMP * systems as each CPU might be scaled differently. So, use the arch * per-CPU loops_per_jiffy value wherever possible. */ -#ifndef CONFIG_SMP -static unsigned long l_p_j_ref; -static unsigned int l_p_j_ref_freq; - static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci) { +#ifndef CONFIG_SMP + static unsigned long l_p_j_ref; + static unsigned int l_p_j_ref_freq; + if (ci->flags & CPUFREQ_CONST_LOOPS) return; if (!l_p_j_ref_freq) { l_p_j_ref = loops_per_jiffy; l_p_j_ref_freq = ci->old; - pr_debug("saving %lu as reference value for loops_per_jiffy; " - "freq is %u kHz\n", l_p_j_ref, l_p_j_ref_freq); + pr_debug("saving %lu as reference value for loops_per_jiffy; freq is %u kHz\n", + l_p_j_ref, l_p_j_ref_freq); } - if ((val == CPUFREQ_POSTCHANGE && ci->old != ci->new) || - (val == CPUFREQ_RESUMECHANGE || val == CPUFREQ_SUSPENDCHANGE)) { + if (val == CPUFREQ_POSTCHANGE && ci->old != ci->new) { loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq, ci->new); - pr_debug("scaling loops_per_jiffy to %lu " - "for frequency %u kHz\n", loops_per_jiffy, ci->new); + pr_debug("scaling loops_per_jiffy to %lu for frequency %u kHz\n", + loops_per_jiffy, ci->new); } -} -#else -static inline void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci) -{ - return; -} #endif +} -static void __cpufreq_notify_transition(struct cpufreq_policy *policy, - struct cpufreq_freqs *freqs, unsigned int state) +/** + * cpufreq_notify_transition - Notify frequency transition and adjust jiffies. + * @policy: cpufreq policy to enable fast frequency switching for. + * @freqs: contain details of the frequency update. + * @state: set to CPUFREQ_PRECHANGE or CPUFREQ_POSTCHANGE. + * + * This function calls the transition notifiers and adjust_jiffies(). + * + * It is called twice on all CPU frequency changes that have external effects. + */ +static void cpufreq_notify_transition(struct cpufreq_policy *policy, + struct cpufreq_freqs *freqs, + unsigned int state) { + int cpu; + BUG_ON(irqs_disabled()); if (cpufreq_disabled()) return; + freqs->policy = policy; freqs->flags = cpufreq_driver->flags; pr_debug("notification %u of frequency transition to %u kHz\n", - state, freqs->new); + state, freqs->new); switch (state) { - case CPUFREQ_PRECHANGE: - if (WARN(policy->transition_ongoing == - cpumask_weight(policy->cpus), - "In middle of another frequency transition\n")) - return; - - policy->transition_ongoing++; - - /* detect if the driver reported a value as "old frequency" + /* + * Detect if the driver reported a value as "old frequency" * which is not equal to what the cpufreq core thinks is * "old frequency". */ - if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) { - if ((policy) && (policy->cpu == freqs->cpu) && - (policy->cur) && (policy->cur != freqs->old)) { - pr_debug("Warning: CPU frequency is" - " %u, cpufreq assumed %u kHz.\n", - freqs->old, policy->cur); - freqs->old = policy->cur; - } + if (policy->cur && policy->cur != freqs->old) { + pr_debug("Warning: CPU frequency is %u, cpufreq assumed %u kHz\n", + freqs->old, policy->cur); + freqs->old = policy->cur; } + srcu_notifier_call_chain(&cpufreq_transition_notifier_list, - CPUFREQ_PRECHANGE, freqs); + CPUFREQ_PRECHANGE, freqs); + adjust_jiffies(CPUFREQ_PRECHANGE, freqs); break; case CPUFREQ_POSTCHANGE: - if (WARN(!policy->transition_ongoing, - "No frequency transition in progress\n")) - return; + adjust_jiffies(CPUFREQ_POSTCHANGE, freqs); + pr_debug("FREQ: %u - CPUs: %*pbl\n", freqs->new, + cpumask_pr_args(policy->cpus)); - policy->transition_ongoing--; + for_each_cpu(cpu, policy->cpus) + trace_cpu_frequency(freqs->new, cpu); - adjust_jiffies(CPUFREQ_POSTCHANGE, freqs); - pr_debug("FREQ: %lu - CPU: %lu", (unsigned long)freqs->new, - (unsigned long)freqs->cpu); - trace_cpu_frequency(freqs->new, freqs->cpu); srcu_notifier_call_chain(&cpufreq_transition_notifier_list, - CPUFREQ_POSTCHANGE, freqs); - if (likely(policy) && likely(policy->cpu == freqs->cpu)) - policy->cur = freqs->new; - break; + CPUFREQ_POSTCHANGE, freqs); + + cpufreq_stats_record_transition(policy, freqs->new); + policy->cur = freqs->new; + } +} + +/* Do post notifications when there are chances that transition has failed */ +static void cpufreq_notify_post_transition(struct cpufreq_policy *policy, + struct cpufreq_freqs *freqs, int transition_failed) +{ + cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE); + if (!transition_failed) + return; + + swap(freqs->old, freqs->new); + cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE); + cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE); +} + +void cpufreq_freq_transition_begin(struct cpufreq_policy *policy, + struct cpufreq_freqs *freqs) +{ + + /* + * Catch double invocations of _begin() which lead to self-deadlock. + * ASYNC_NOTIFICATION drivers are left out because the cpufreq core + * doesn't invoke _begin() on their behalf, and hence the chances of + * double invocations are very low. Moreover, there are scenarios + * where these checks can emit false-positive warnings in these + * drivers; so we avoid that by skipping them altogether. + */ + WARN_ON(!(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION) + && current == policy->transition_task); + +wait: + wait_event(policy->transition_wait, !policy->transition_ongoing); + + spin_lock(&policy->transition_lock); + + if (unlikely(policy->transition_ongoing)) { + spin_unlock(&policy->transition_lock); + goto wait; + } + + policy->transition_ongoing = true; + policy->transition_task = current; + + spin_unlock(&policy->transition_lock); + + cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE); +} +EXPORT_SYMBOL_GPL(cpufreq_freq_transition_begin); + +void cpufreq_freq_transition_end(struct cpufreq_policy *policy, + struct cpufreq_freqs *freqs, int transition_failed) +{ + if (WARN_ON(!policy->transition_ongoing)) + return; + + cpufreq_notify_post_transition(policy, freqs, transition_failed); + + arch_set_freq_scale(policy->related_cpus, + policy->cur, + arch_scale_freq_ref(policy->cpu)); + + spin_lock(&policy->transition_lock); + policy->transition_ongoing = false; + policy->transition_task = NULL; + spin_unlock(&policy->transition_lock); + + wake_up(&policy->transition_wait); +} +EXPORT_SYMBOL_GPL(cpufreq_freq_transition_end); + +/* + * Fast frequency switching status count. Positive means "enabled", negative + * means "disabled" and 0 means "not decided yet". + */ +static int cpufreq_fast_switch_count; +static DEFINE_MUTEX(cpufreq_fast_switch_lock); + +static void cpufreq_list_transition_notifiers(void) +{ + struct notifier_block *nb; + + pr_info("Registered transition notifiers:\n"); + + mutex_lock(&cpufreq_transition_notifier_list.mutex); + + for (nb = cpufreq_transition_notifier_list.head; nb; nb = nb->next) + pr_info("%pS\n", nb->notifier_call); + + mutex_unlock(&cpufreq_transition_notifier_list.mutex); +} + +/** + * cpufreq_enable_fast_switch - Enable fast frequency switching for policy. + * @policy: cpufreq policy to enable fast frequency switching for. + * + * Try to enable fast frequency switching for @policy. + * + * The attempt will fail if there is at least one transition notifier registered + * at this point, as fast frequency switching is quite fundamentally at odds + * with transition notifiers. Thus if successful, it will make registration of + * transition notifiers fail going forward. + */ +void cpufreq_enable_fast_switch(struct cpufreq_policy *policy) +{ + lockdep_assert_held(&policy->rwsem); + + if (!policy->fast_switch_possible) + return; + + mutex_lock(&cpufreq_fast_switch_lock); + if (cpufreq_fast_switch_count >= 0) { + cpufreq_fast_switch_count++; + policy->fast_switch_enabled = true; + } else { + pr_warn("CPU%u: Fast frequency switching not enabled\n", + policy->cpu); + cpufreq_list_transition_notifiers(); + } + mutex_unlock(&cpufreq_fast_switch_lock); +} +EXPORT_SYMBOL_GPL(cpufreq_enable_fast_switch); + +/** + * cpufreq_disable_fast_switch - Disable fast frequency switching for policy. + * @policy: cpufreq policy to disable fast frequency switching for. + */ +void cpufreq_disable_fast_switch(struct cpufreq_policy *policy) +{ + mutex_lock(&cpufreq_fast_switch_lock); + if (policy->fast_switch_enabled) { + policy->fast_switch_enabled = false; + if (!WARN_ON(cpufreq_fast_switch_count <= 0)) + cpufreq_fast_switch_count--; } + mutex_unlock(&cpufreq_fast_switch_lock); +} +EXPORT_SYMBOL_GPL(cpufreq_disable_fast_switch); + +static unsigned int __resolve_freq(struct cpufreq_policy *policy, + unsigned int target_freq, + unsigned int min, unsigned int max, + unsigned int relation) +{ + unsigned int idx; + + target_freq = clamp_val(target_freq, min, max); + + if (!policy->freq_table) + return target_freq; + + idx = cpufreq_frequency_table_target(policy, target_freq, min, max, relation); + policy->cached_resolved_idx = idx; + policy->cached_target_freq = target_freq; + return policy->freq_table[idx].frequency; } /** - * cpufreq_notify_transition - call notifier chain and adjust_jiffies - * on frequency transition. + * cpufreq_driver_resolve_freq - Map a target frequency to a driver-supported + * one. + * @policy: associated policy to interrogate + * @target_freq: target frequency to resolve. + * + * The target to driver frequency mapping is cached in the policy. * - * This function calls the transition notifiers and the "adjust_jiffies" - * function. It is called twice on all CPU frequency changes that have - * external effects. + * Return: Lowest driver-supported frequency greater than or equal to the + * given target_freq, subject to policy (min/max) and driver limitations. */ -void cpufreq_notify_transition(struct cpufreq_policy *policy, - struct cpufreq_freqs *freqs, unsigned int state) +unsigned int cpufreq_driver_resolve_freq(struct cpufreq_policy *policy, + unsigned int target_freq) { - for_each_cpu(freqs->cpu, policy->cpus) - __cpufreq_notify_transition(policy, freqs, state); + unsigned int min = READ_ONCE(policy->min); + unsigned int max = READ_ONCE(policy->max); + + /* + * If this function runs in parallel with cpufreq_set_policy(), it may + * read policy->min before the update and policy->max after the update + * or the other way around, so there is no ordering guarantee. + * + * Resolve this by always honoring the max (in case it comes from + * thermal throttling or similar). + */ + if (unlikely(min > max)) + min = max; + + return __resolve_freq(policy, target_freq, min, max, CPUFREQ_RELATION_LE); } -EXPORT_SYMBOL_GPL(cpufreq_notify_transition); +EXPORT_SYMBOL_GPL(cpufreq_driver_resolve_freq); +unsigned int cpufreq_policy_transition_delay_us(struct cpufreq_policy *policy) +{ + unsigned int latency; + + if (policy->transition_delay_us) + return policy->transition_delay_us; + + latency = policy->cpuinfo.transition_latency / NSEC_PER_USEC; + if (latency) + /* Give a 50% breathing room between updates */ + return latency + (latency >> 1); + + return USEC_PER_MSEC; +} +EXPORT_SYMBOL_GPL(cpufreq_policy_transition_delay_us); /********************************************************************* * SYSFS INTERFACE * *********************************************************************/ +static ssize_t show_boost(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%d\n", cpufreq_driver->boost_enabled); +} + +static ssize_t store_boost(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t count) +{ + bool enable; + + if (kstrtobool(buf, &enable)) + return -EINVAL; + + if (cpufreq_boost_trigger_state(enable)) { + pr_err("%s: Cannot %s BOOST!\n", + __func__, str_enable_disable(enable)); + return -EINVAL; + } + + pr_debug("%s: cpufreq BOOST %s\n", + __func__, str_enabled_disabled(enable)); + + return count; +} +define_one_global_rw(boost); + +static ssize_t show_local_boost(struct cpufreq_policy *policy, char *buf) +{ + return sysfs_emit(buf, "%d\n", policy->boost_enabled); +} + +static int policy_set_boost(struct cpufreq_policy *policy, bool enable) +{ + int ret; + + if (policy->boost_enabled == enable) + return 0; + + policy->boost_enabled = enable; + + ret = cpufreq_driver->set_boost(policy, enable); + if (ret) + policy->boost_enabled = !policy->boost_enabled; + + return ret; +} -static struct cpufreq_governor *__find_governor(const char *str_governor) +static ssize_t store_local_boost(struct cpufreq_policy *policy, + const char *buf, size_t count) +{ + int ret; + bool enable; + + if (kstrtobool(buf, &enable)) + return -EINVAL; + + if (!cpufreq_driver->boost_enabled) + return -EINVAL; + + if (!policy->boost_supported) + return -EINVAL; + + ret = policy_set_boost(policy, enable); + if (!ret) + return count; + + return ret; +} + +static struct freq_attr local_boost = __ATTR(boost, 0644, show_local_boost, store_local_boost); + +static struct cpufreq_governor *find_governor(const char *str_governor) { struct cpufreq_governor *t; - list_for_each_entry(t, &cpufreq_governor_list, governor_list) - if (!strnicmp(str_governor, t->name, CPUFREQ_NAME_LEN)) + for_each_governor(t) + if (!strncasecmp(str_governor, t->name, CPUFREQ_NAME_LEN)) return t; return NULL; } -/** - * cpufreq_parse_governor - parse a governor string - */ -static int cpufreq_parse_governor(char *str_governor, unsigned int *policy, - struct cpufreq_governor **governor) +static struct cpufreq_governor *get_governor(const char *str_governor) { - int err = -EINVAL; + struct cpufreq_governor *t; - if (!cpufreq_driver) - goto out; + mutex_lock(&cpufreq_governor_mutex); + t = find_governor(str_governor); + if (!t) + goto unlock; - if (cpufreq_driver->setpolicy) { - if (!strnicmp(str_governor, "performance", CPUFREQ_NAME_LEN)) { - *policy = CPUFREQ_POLICY_PERFORMANCE; - err = 0; - } else if (!strnicmp(str_governor, "powersave", - CPUFREQ_NAME_LEN)) { - *policy = CPUFREQ_POLICY_POWERSAVE; - err = 0; - } - } else if (cpufreq_driver->target) { - struct cpufreq_governor *t; + if (!try_module_get(t->owner)) + t = NULL; + +unlock: + mutex_unlock(&cpufreq_governor_mutex); - mutex_lock(&cpufreq_governor_mutex); + return t; +} - t = __find_governor(str_governor); +static unsigned int cpufreq_parse_policy(char *str_governor) +{ + if (!strncasecmp(str_governor, "performance", strlen("performance"))) + return CPUFREQ_POLICY_PERFORMANCE; - if (t == NULL) { - int ret; + if (!strncasecmp(str_governor, "powersave", strlen("powersave"))) + return CPUFREQ_POLICY_POWERSAVE; - mutex_unlock(&cpufreq_governor_mutex); - ret = request_module("cpufreq_%s", str_governor); - mutex_lock(&cpufreq_governor_mutex); + return CPUFREQ_POLICY_UNKNOWN; +} - if (ret == 0) - t = __find_governor(str_governor); - } +/** + * cpufreq_parse_governor - parse a governor string only for has_target() + * @str_governor: Governor name. + */ +static struct cpufreq_governor *cpufreq_parse_governor(char *str_governor) +{ + struct cpufreq_governor *t; - if (t != NULL) { - *governor = t; - err = 0; - } + t = get_governor(str_governor); + if (t) + return t; - mutex_unlock(&cpufreq_governor_mutex); - } -out: - return err; + if (request_module("cpufreq_%s", str_governor)) + return NULL; + + return get_governor(str_governor); } -/** +/* * cpufreq_per_cpu_attr_read() / show_##file_name() - * print out cpufreq information * @@ -449,7 +703,7 @@ out: static ssize_t show_##file_name \ (struct cpufreq_policy *policy, char *buf) \ { \ - return sprintf(buf, "%u\n", policy->object); \ + return sysfs_emit(buf, "%u\n", policy->object); \ } show_one(cpuinfo_min_freq, cpuinfo.min_freq); @@ -457,103 +711,135 @@ show_one(cpuinfo_max_freq, cpuinfo.max_freq); show_one(cpuinfo_transition_latency, cpuinfo.transition_latency); show_one(scaling_min_freq, min); show_one(scaling_max_freq, max); -show_one(scaling_cur_freq, cur); -static int __cpufreq_set_policy(struct cpufreq_policy *data, - struct cpufreq_policy *policy); +__weak int arch_freq_get_on_cpu(int cpu) +{ + return -EOPNOTSUPP; +} -/** +static inline bool cpufreq_avg_freq_supported(struct cpufreq_policy *policy) +{ + return arch_freq_get_on_cpu(policy->cpu) != -EOPNOTSUPP; +} + +static ssize_t show_scaling_cur_freq(struct cpufreq_policy *policy, char *buf) +{ + ssize_t ret; + int freq; + + freq = IS_ENABLED(CONFIG_CPUFREQ_ARCH_CUR_FREQ) + ? arch_freq_get_on_cpu(policy->cpu) + : 0; + + if (freq > 0) + ret = sysfs_emit(buf, "%u\n", freq); + else if (cpufreq_driver->setpolicy && cpufreq_driver->get) + ret = sysfs_emit(buf, "%u\n", cpufreq_driver->get(policy->cpu)); + else + ret = sysfs_emit(buf, "%u\n", policy->cur); + return ret; +} + +/* * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access */ #define store_one(file_name, object) \ static ssize_t store_##file_name \ (struct cpufreq_policy *policy, const char *buf, size_t count) \ { \ - unsigned int ret; \ - struct cpufreq_policy new_policy; \ + unsigned long val; \ + int ret; \ \ - ret = cpufreq_get_policy(&new_policy, policy->cpu); \ + ret = kstrtoul(buf, 0, &val); \ if (ret) \ - return -EINVAL; \ - \ - ret = sscanf(buf, "%u", &new_policy.object); \ - if (ret != 1) \ - return -EINVAL; \ + return ret; \ \ - ret = __cpufreq_set_policy(policy, &new_policy); \ - policy->user_policy.object = policy->object; \ - \ - return ret ? ret : count; \ + ret = freq_qos_update_request(policy->object##_freq_req, val);\ + return ret >= 0 ? count : ret; \ } store_one(scaling_min_freq, min); store_one(scaling_max_freq, max); -/** +/* * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware */ static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy, char *buf) { - unsigned int cur_freq = __cpufreq_get(policy->cpu); - if (!cur_freq) - return sprintf(buf, "<unknown>"); - return sprintf(buf, "%u\n", cur_freq); + unsigned int cur_freq = __cpufreq_get(policy); + + if (cur_freq) + return sysfs_emit(buf, "%u\n", cur_freq); + + return sysfs_emit(buf, "<unknown>\n"); +} + +/* + * show_cpuinfo_avg_freq - average CPU frequency as detected by hardware + */ +static ssize_t show_cpuinfo_avg_freq(struct cpufreq_policy *policy, + char *buf) +{ + int avg_freq = arch_freq_get_on_cpu(policy->cpu); + + if (avg_freq > 0) + return sysfs_emit(buf, "%u\n", avg_freq); + return avg_freq != 0 ? avg_freq : -EINVAL; } -/** +/* * show_scaling_governor - show the current policy for the specified CPU */ static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf) { if (policy->policy == CPUFREQ_POLICY_POWERSAVE) - return sprintf(buf, "powersave\n"); + return sysfs_emit(buf, "powersave\n"); else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE) - return sprintf(buf, "performance\n"); + return sysfs_emit(buf, "performance\n"); else if (policy->governor) - return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", - policy->governor->name); + return sysfs_emit(buf, "%s\n", policy->governor->name); return -EINVAL; } -/** +/* * store_scaling_governor - store policy for the specified CPU */ static ssize_t store_scaling_governor(struct cpufreq_policy *policy, const char *buf, size_t count) { - unsigned int ret; - char str_governor[16]; - struct cpufreq_policy new_policy; - - ret = cpufreq_get_policy(&new_policy, policy->cpu); - if (ret) - return ret; + char str_governor[CPUFREQ_NAME_LEN]; + int ret; ret = sscanf(buf, "%15s", str_governor); if (ret != 1) return -EINVAL; - if (cpufreq_parse_governor(str_governor, &new_policy.policy, - &new_policy.governor)) - return -EINVAL; + if (cpufreq_driver->setpolicy) { + unsigned int new_pol; - /* - * Do not use cpufreq_set_policy here or the user_policy.max - * will be wrongly overridden - */ - ret = __cpufreq_set_policy(policy, &new_policy); + new_pol = cpufreq_parse_policy(str_governor); + if (!new_pol) + return -EINVAL; - policy->user_policy.policy = policy->policy; - policy->user_policy.governor = policy->governor; + ret = cpufreq_set_policy(policy, NULL, new_pol); + } else { + struct cpufreq_governor *new_gov; - if (ret) - return ret; - else - return count; + new_gov = cpufreq_parse_governor(str_governor); + if (!new_gov) + return -EINVAL; + + ret = cpufreq_set_policy(policy, new_gov, + CPUFREQ_POLICY_UNKNOWN); + + module_put(new_gov->owner); + } + + return ret ? ret : count; } -/** +/* * show_scaling_driver - show the cpufreq driver currently loaded */ static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf) @@ -561,7 +847,7 @@ static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf) return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name); } -/** +/* * show_scaling_available_governors - show the available CPUfreq governors */ static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy, @@ -570,19 +856,21 @@ static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy, ssize_t i = 0; struct cpufreq_governor *t; - if (!cpufreq_driver->target) { - i += sprintf(buf, "performance powersave"); + if (!has_target()) { + i += sysfs_emit(buf, "performance powersave"); goto out; } - list_for_each_entry(t, &cpufreq_governor_list, governor_list) { + mutex_lock(&cpufreq_governor_mutex); + for_each_governor(t) { if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char)) - (CPUFREQ_NAME_LEN + 2))) - goto out; - i += scnprintf(&buf[i], CPUFREQ_NAME_PLEN, "%s ", t->name); + break; + i += sysfs_emit_at(buf, i, "%s ", t->name); } + mutex_unlock(&cpufreq_governor_mutex); out: - i += sprintf(&buf[i], "\n"); + i += sysfs_emit_at(buf, i, "\n"); return i; } @@ -592,18 +880,20 @@ ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf) unsigned int cpu; for_each_cpu(cpu, mask) { - if (i) - i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " "); - i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu); + i += sysfs_emit_at(buf, i, "%u ", cpu); if (i >= (PAGE_SIZE - 5)) break; } - i += sprintf(&buf[i], "\n"); + + /* Remove the extra space at the end */ + i--; + + i += sysfs_emit_at(buf, i, "\n"); return i; } EXPORT_SYMBOL_GPL(cpufreq_show_cpus); -/** +/* * show_related_cpus - show the CPUs affected by each transition even if * hw coordination is in use */ @@ -612,7 +902,7 @@ static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf) return cpufreq_show_cpus(policy->related_cpus, buf); } -/** +/* * show_affected_cpus - show the CPUs affected by each transition */ static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf) @@ -624,14 +914,14 @@ static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy, const char *buf, size_t count) { unsigned int freq = 0; - unsigned int ret; + int ret; if (!policy->governor || !policy->governor->store_setspeed) return -EINVAL; - ret = sscanf(buf, "%u", &freq); - if (ret != 1) - return -EINVAL; + ret = kstrtouint(buf, 0, &freq); + if (ret) + return ret; policy->governor->store_setspeed(policy, freq); @@ -641,27 +931,26 @@ static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy, static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf) { if (!policy->governor || !policy->governor->show_setspeed) - return sprintf(buf, "<unsupported>\n"); + return sysfs_emit(buf, "<unsupported>\n"); return policy->governor->show_setspeed(policy, buf); } -/** +/* * show_bios_limit - show the current cpufreq HW/BIOS limitation */ static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf) { unsigned int limit; int ret; - if (cpufreq_driver->bios_limit) { - ret = cpufreq_driver->bios_limit(policy->cpu, &limit); - if (!ret) - return sprintf(buf, "%u\n", limit); - } - return sprintf(buf, "%u\n", policy->cpuinfo.max_freq); + ret = cpufreq_driver->bios_limit(policy->cpu, &limit); + if (!ret) + return sysfs_emit(buf, "%u\n", limit); + return sysfs_emit(buf, "%u\n", policy->cpuinfo.max_freq); } cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400); +cpufreq_freq_attr_ro(cpuinfo_avg_freq); cpufreq_freq_attr_ro(cpuinfo_min_freq); cpufreq_freq_attr_ro(cpuinfo_max_freq); cpufreq_freq_attr_ro(cpuinfo_transition_latency); @@ -676,10 +965,11 @@ cpufreq_freq_attr_rw(scaling_max_freq); cpufreq_freq_attr_rw(scaling_governor); cpufreq_freq_attr_rw(scaling_setspeed); -static struct attribute *default_attrs[] = { +static struct attribute *cpufreq_attrs[] = { &cpuinfo_min_freq.attr, &cpuinfo_max_freq.attr, &cpuinfo_transition_latency.attr, + &scaling_cur_freq.attr, &scaling_min_freq.attr, &scaling_max_freq.attr, &affected_cpus.attr, @@ -690,6 +980,7 @@ static struct attribute *default_attrs[] = { &scaling_setspeed.attr, NULL }; +ATTRIBUTE_GROUPS(cpufreq); #define to_policy(k) container_of(k, struct cpufreq_policy, kobj) #define to_attr(a) container_of(a, struct freq_attr, attr) @@ -698,24 +989,16 @@ static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf) { struct cpufreq_policy *policy = to_policy(kobj); struct freq_attr *fattr = to_attr(attr); - ssize_t ret = -EINVAL; - policy = cpufreq_cpu_get_sysfs(policy->cpu); - if (!policy) - goto no_policy; - if (lock_policy_rwsem_read(policy->cpu) < 0) - goto fail; + if (!fattr->show) + return -EIO; - if (fattr->show) - ret = fattr->show(policy, buf); - else - ret = -EIO; + guard(cpufreq_policy_read)(policy); - unlock_policy_rwsem_read(policy->cpu); -fail: - cpufreq_cpu_put_sysfs(policy); -no_policy: - return ret; + if (likely(!policy_is_inactive(policy))) + return fattr->show(policy, buf); + + return -EBUSY; } static ssize_t store(struct kobject *kobj, struct attribute *attr, @@ -723,24 +1006,16 @@ static ssize_t store(struct kobject *kobj, struct attribute *attr, { struct cpufreq_policy *policy = to_policy(kobj); struct freq_attr *fattr = to_attr(attr); - ssize_t ret = -EINVAL; - policy = cpufreq_cpu_get_sysfs(policy->cpu); - if (!policy) - goto no_policy; - if (lock_policy_rwsem_write(policy->cpu) < 0) - goto fail; + if (!fattr->store) + return -EIO; - if (fattr->store) - ret = fattr->store(policy, buf, count); - else - ret = -EIO; + guard(cpufreq_policy_write)(policy); - unlock_policy_rwsem_write(policy->cpu); -fail: - cpufreq_cpu_put_sysfs(policy); -no_policy: - return ret; + if (likely(!policy_is_inactive(policy))) + return fattr->store(policy, buf, count); + + return -EBUSY; } static void cpufreq_sysfs_release(struct kobject *kobj) @@ -755,251 +1030,236 @@ static const struct sysfs_ops sysfs_ops = { .store = store, }; -static struct kobj_type ktype_cpufreq = { +static const struct kobj_type ktype_cpufreq = { .sysfs_ops = &sysfs_ops, - .default_attrs = default_attrs, + .default_groups = cpufreq_groups, .release = cpufreq_sysfs_release, }; -struct kobject *cpufreq_global_kobject; -EXPORT_SYMBOL(cpufreq_global_kobject); - -static int cpufreq_global_kobject_usage; - -int cpufreq_get_global_kobject(void) -{ - if (!cpufreq_global_kobject_usage++) - return kobject_add(cpufreq_global_kobject, - &cpu_subsys.dev_root->kobj, "%s", "cpufreq"); - - return 0; -} -EXPORT_SYMBOL(cpufreq_get_global_kobject); - -void cpufreq_put_global_kobject(void) -{ - if (!--cpufreq_global_kobject_usage) - kobject_del(cpufreq_global_kobject); -} -EXPORT_SYMBOL(cpufreq_put_global_kobject); - -int cpufreq_sysfs_create_file(const struct attribute *attr) +static void add_cpu_dev_symlink(struct cpufreq_policy *policy, unsigned int cpu, + struct device *dev) { - int ret = cpufreq_get_global_kobject(); + if (unlikely(!dev)) + return; - if (!ret) { - ret = sysfs_create_file(cpufreq_global_kobject, attr); - if (ret) - cpufreq_put_global_kobject(); - } + if (cpumask_test_and_set_cpu(cpu, policy->real_cpus)) + return; - return ret; + dev_dbg(dev, "%s: Adding symlink\n", __func__); + if (sysfs_create_link(&dev->kobj, &policy->kobj, "cpufreq")) + dev_err(dev, "cpufreq symlink creation failed\n"); } -EXPORT_SYMBOL(cpufreq_sysfs_create_file); -void cpufreq_sysfs_remove_file(const struct attribute *attr) +static void remove_cpu_dev_symlink(struct cpufreq_policy *policy, int cpu, + struct device *dev) { - sysfs_remove_file(cpufreq_global_kobject, attr); - cpufreq_put_global_kobject(); + dev_dbg(dev, "%s: Removing symlink\n", __func__); + sysfs_remove_link(&dev->kobj, "cpufreq"); + cpumask_clear_cpu(cpu, policy->real_cpus); } -EXPORT_SYMBOL(cpufreq_sysfs_remove_file); -/* symlink affected CPUs */ -static int cpufreq_add_dev_symlink(unsigned int cpu, - struct cpufreq_policy *policy) +static int cpufreq_add_dev_interface(struct cpufreq_policy *policy) { - unsigned int j; + struct freq_attr **drv_attr; int ret = 0; - for_each_cpu(j, policy->cpus) { - struct cpufreq_policy *managed_policy; - struct device *cpu_dev; - - if (j == cpu) - continue; - - pr_debug("CPU %u already managed, adding link\n", j); - managed_policy = cpufreq_cpu_get(cpu); - cpu_dev = get_cpu_device(j); - ret = sysfs_create_link(&cpu_dev->kobj, &policy->kobj, - "cpufreq"); - if (ret) { - cpufreq_cpu_put(managed_policy); + /* Attributes that need freq_table */ + if (policy->freq_table) { + ret = sysfs_create_file(&policy->kobj, + &cpufreq_freq_attr_scaling_available_freqs.attr); + if (ret) return ret; + + if (cpufreq_boost_supported()) { + ret = sysfs_create_file(&policy->kobj, + &cpufreq_freq_attr_scaling_boost_freqs.attr); + if (ret) + return ret; } } - return ret; -} - -static int cpufreq_add_dev_interface(unsigned int cpu, - struct cpufreq_policy *policy, - struct device *dev) -{ - struct cpufreq_policy new_policy; - struct freq_attr **drv_attr; - unsigned long flags; - int ret = 0; - unsigned int j; - - /* prepare interface data */ - ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq, - &dev->kobj, "cpufreq"); - if (ret) - return ret; /* set up files for this cpu device */ drv_attr = cpufreq_driver->attr; - while ((drv_attr) && (*drv_attr)) { + while (drv_attr && *drv_attr) { ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr)); if (ret) - goto err_out_kobj_put; + return ret; drv_attr++; } if (cpufreq_driver->get) { ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr); if (ret) - goto err_out_kobj_put; + return ret; } - if (cpufreq_driver->target) { - ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr); + + if (cpufreq_avg_freq_supported(policy)) { + ret = sysfs_create_file(&policy->kobj, &cpuinfo_avg_freq.attr); if (ret) - goto err_out_kobj_put; + return ret; } + if (cpufreq_driver->bios_limit) { ret = sysfs_create_file(&policy->kobj, &bios_limit.attr); if (ret) - goto err_out_kobj_put; + return ret; } - write_lock_irqsave(&cpufreq_driver_lock, flags); - for_each_cpu(j, policy->cpus) { - per_cpu(cpufreq_cpu_data, j) = policy; - per_cpu(cpufreq_policy_cpu, j) = policy->cpu; + if (cpufreq_boost_supported()) { + ret = sysfs_create_file(&policy->kobj, &local_boost.attr); + if (ret) + return ret; } - write_unlock_irqrestore(&cpufreq_driver_lock, flags); - ret = cpufreq_add_dev_symlink(cpu, policy); - if (ret) - goto err_out_kobj_put; + return 0; +} - memcpy(&new_policy, policy, sizeof(struct cpufreq_policy)); - /* assure that the starting sequence is run in __cpufreq_set_policy */ - policy->governor = NULL; +static int cpufreq_init_policy(struct cpufreq_policy *policy) +{ + struct cpufreq_governor *gov = NULL; + unsigned int pol = CPUFREQ_POLICY_UNKNOWN; + int ret; - /* set default policy */ - ret = __cpufreq_set_policy(policy, &new_policy); - policy->user_policy.policy = policy->policy; - policy->user_policy.governor = policy->governor; + if (has_target()) { + /* Update policy governor to the one used before hotplug. */ + if (policy->last_governor[0] != '\0') + gov = get_governor(policy->last_governor); + if (gov) { + pr_debug("Restoring governor %s for cpu %d\n", + gov->name, policy->cpu); + } else { + gov = get_governor(default_governor); + } - if (ret) { - pr_debug("setting policy failed\n"); - if (cpufreq_driver->exit) - cpufreq_driver->exit(policy); + if (!gov) { + gov = cpufreq_default_governor(); + __module_get(gov->owner); + } + + } else { + + /* Use the default policy if there is no last_policy. */ + if (policy->last_policy) { + pol = policy->last_policy; + } else { + pol = cpufreq_parse_policy(default_governor); + /* + * In case the default governor is neither "performance" + * nor "powersave", fall back to the initial policy + * value set by the driver. + */ + if (pol == CPUFREQ_POLICY_UNKNOWN) + pol = policy->policy; + } + if (pol != CPUFREQ_POLICY_PERFORMANCE && + pol != CPUFREQ_POLICY_POWERSAVE) + return -ENODATA; } - return ret; -err_out_kobj_put: - kobject_put(&policy->kobj); - wait_for_completion(&policy->kobj_unregister); + ret = cpufreq_set_policy(policy, gov, pol); + if (gov) + module_put(gov->owner); + return ret; } -#ifdef CONFIG_HOTPLUG_CPU -static int cpufreq_add_policy_cpu(unsigned int cpu, unsigned int sibling, - struct device *dev) +static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu) { - struct cpufreq_policy *policy; - int ret = 0, has_target = !!cpufreq_driver->target; - unsigned long flags; - - policy = cpufreq_cpu_get(sibling); - WARN_ON(!policy); + int ret = 0; - if (has_target) - __cpufreq_governor(policy, CPUFREQ_GOV_STOP); + /* Has this CPU been taken care of already? */ + if (cpumask_test_cpu(cpu, policy->cpus)) + return 0; - lock_policy_rwsem_write(sibling); + guard(cpufreq_policy_write)(policy); - write_lock_irqsave(&cpufreq_driver_lock, flags); + if (has_target()) + cpufreq_stop_governor(policy); cpumask_set_cpu(cpu, policy->cpus); - per_cpu(cpufreq_policy_cpu, cpu) = policy->cpu; - per_cpu(cpufreq_cpu_data, cpu) = policy; - write_unlock_irqrestore(&cpufreq_driver_lock, flags); - - unlock_policy_rwsem_write(sibling); - if (has_target) { - __cpufreq_governor(policy, CPUFREQ_GOV_START); - __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS); + if (has_target()) { + ret = cpufreq_start_governor(policy); + if (ret) + pr_err("%s: Failed to start governor\n", __func__); } - ret = sysfs_create_link(&dev->kobj, &policy->kobj, "cpufreq"); - if (ret) { - cpufreq_cpu_put(policy); - return ret; + return ret; +} + +void refresh_frequency_limits(struct cpufreq_policy *policy) +{ + if (!policy_is_inactive(policy)) { + pr_debug("updating policy for CPU %u\n", policy->cpu); + + cpufreq_set_policy(policy, policy->governor, policy->policy); } +} +EXPORT_SYMBOL(refresh_frequency_limits); + +static void handle_update(struct work_struct *work) +{ + struct cpufreq_policy *policy = + container_of(work, struct cpufreq_policy, update); + + pr_debug("handle_update for cpu %u called\n", policy->cpu); + + guard(cpufreq_policy_write)(policy); + + refresh_frequency_limits(policy); +} +static int cpufreq_notifier_min(struct notifier_block *nb, unsigned long freq, + void *data) +{ + struct cpufreq_policy *policy = container_of(nb, struct cpufreq_policy, nb_min); + + schedule_work(&policy->update); return 0; } -#endif -/** - * cpufreq_add_dev - add a CPU device - * - * Adds the cpufreq interface for a CPU device. - * - * The Oracle says: try running cpufreq registration/unregistration concurrently - * with with cpu hotplugging and all hell will break loose. Tried to clean this - * mess up, but more thorough testing is needed. - Mathieu - */ -static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif) +static int cpufreq_notifier_max(struct notifier_block *nb, unsigned long freq, + void *data) { - unsigned int j, cpu = dev->id; - int ret = -ENOMEM; - struct cpufreq_policy *policy; - unsigned long flags; -#ifdef CONFIG_HOTPLUG_CPU - struct cpufreq_governor *gov; - int sibling; -#endif + struct cpufreq_policy *policy = container_of(nb, struct cpufreq_policy, nb_max); - if (cpu_is_offline(cpu)) - return 0; + schedule_work(&policy->update); + return 0; +} - pr_debug("adding CPU %u\n", cpu); +static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy) +{ + struct kobject *kobj; + struct completion *cmp; -#ifdef CONFIG_SMP - /* check whether a different CPU already registered this - * CPU because it is in the same boat. */ - policy = cpufreq_cpu_get(cpu); - if (unlikely(policy)) { - cpufreq_cpu_put(policy); - return 0; + scoped_guard(cpufreq_policy_write, policy) { + cpufreq_stats_free_table(policy); + kobj = &policy->kobj; + cmp = &policy->kobj_unregister; } + kobject_put(kobj); -#ifdef CONFIG_HOTPLUG_CPU - /* Check if this cpu was hot-unplugged earlier and has siblings */ - read_lock_irqsave(&cpufreq_driver_lock, flags); - for_each_online_cpu(sibling) { - struct cpufreq_policy *cp = per_cpu(cpufreq_cpu_data, sibling); - if (cp && cpumask_test_cpu(cpu, cp->related_cpus)) { - read_unlock_irqrestore(&cpufreq_driver_lock, flags); - return cpufreq_add_policy_cpu(cpu, sibling, dev); - } - } - read_unlock_irqrestore(&cpufreq_driver_lock, flags); -#endif -#endif + /* + * We need to make sure that the underlying kobj is + * actually not referenced anymore by anybody before we + * proceed with unloading. + */ + pr_debug("waiting for dropping of refcount\n"); + wait_for_completion(cmp); + pr_debug("wait complete\n"); +} - if (!try_module_get(cpufreq_driver->owner)) { - ret = -EINVAL; - goto module_out; - } +static struct cpufreq_policy *cpufreq_policy_alloc(unsigned int cpu) +{ + struct cpufreq_policy *policy; + struct device *dev = get_cpu_device(cpu); + int ret; - policy = kzalloc(sizeof(struct cpufreq_policy), GFP_KERNEL); + if (!dev) + return NULL; + + policy = kzalloc(sizeof(*policy), GFP_KERNEL); if (!policy) - goto nomem_out; + return NULL; if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL)) goto err_free_policy; @@ -1007,263 +1267,576 @@ static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif) if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL)) goto err_free_cpumask; - policy->cpu = cpu; - policy->governor = CPUFREQ_DEFAULT_GOVERNOR; - cpumask_copy(policy->cpus, cpumask_of(cpu)); - - /* Initially set CPU itself as the policy_cpu */ - per_cpu(cpufreq_policy_cpu, cpu) = cpu; + if (!zalloc_cpumask_var(&policy->real_cpus, GFP_KERNEL)) + goto err_free_rcpumask; init_completion(&policy->kobj_unregister); - INIT_WORK(&policy->update, handle_update); - - /* call driver. From then on the cpufreq must be able - * to accept all calls to ->verify and ->setpolicy for this CPU - */ - ret = cpufreq_driver->init(policy); + ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq, + cpufreq_global_kobject, "policy%u", cpu); if (ret) { - pr_debug("initialization failed\n"); - goto err_set_policy_cpu; + dev_err(dev, "%s: failed to init policy->kobj: %d\n", __func__, ret); + /* + * The entire policy object will be freed below, but the extra + * memory allocated for the kobject name needs to be freed by + * releasing the kobject. + */ + kobject_put(&policy->kobj); + goto err_free_real_cpus; } - /* related cpus should atleast have policy->cpus */ - cpumask_or(policy->related_cpus, policy->related_cpus, policy->cpus); + init_rwsem(&policy->rwsem); - /* - * affected cpus must always be the one, which are online. We aren't - * managing offline cpus here. - */ - cpumask_and(policy->cpus, policy->cpus, cpu_online_mask); + freq_constraints_init(&policy->constraints); - policy->user_policy.min = policy->min; - policy->user_policy.max = policy->max; + policy->nb_min.notifier_call = cpufreq_notifier_min; + policy->nb_max.notifier_call = cpufreq_notifier_max; - blocking_notifier_call_chain(&cpufreq_policy_notifier_list, - CPUFREQ_START, policy); + ret = freq_qos_add_notifier(&policy->constraints, FREQ_QOS_MIN, + &policy->nb_min); + if (ret) { + dev_err(dev, "Failed to register MIN QoS notifier: %d (CPU%u)\n", + ret, cpu); + goto err_kobj_remove; + } -#ifdef CONFIG_HOTPLUG_CPU - gov = __find_governor(per_cpu(cpufreq_cpu_governor, cpu)); - if (gov) { - policy->governor = gov; - pr_debug("Restoring governor %s for cpu %d\n", - policy->governor->name, cpu); + ret = freq_qos_add_notifier(&policy->constraints, FREQ_QOS_MAX, + &policy->nb_max); + if (ret) { + dev_err(dev, "Failed to register MAX QoS notifier: %d (CPU%u)\n", + ret, cpu); + goto err_min_qos_notifier; } -#endif - ret = cpufreq_add_dev_interface(cpu, policy, dev); - if (ret) - goto err_out_unregister; + INIT_LIST_HEAD(&policy->policy_list); + spin_lock_init(&policy->transition_lock); + init_waitqueue_head(&policy->transition_wait); + INIT_WORK(&policy->update, handle_update); - kobject_uevent(&policy->kobj, KOBJ_ADD); - module_put(cpufreq_driver->owner); - pr_debug("initialization complete\n"); + return policy; - return 0; +err_min_qos_notifier: + freq_qos_remove_notifier(&policy->constraints, FREQ_QOS_MIN, + &policy->nb_min); +err_kobj_remove: + cpufreq_policy_put_kobj(policy); +err_free_real_cpus: + free_cpumask_var(policy->real_cpus); +err_free_rcpumask: + free_cpumask_var(policy->related_cpus); +err_free_cpumask: + free_cpumask_var(policy->cpus); +err_free_policy: + kfree(policy); + + return NULL; +} + +static void cpufreq_policy_free(struct cpufreq_policy *policy) +{ + unsigned long flags; + int cpu; -err_out_unregister: + /* + * The callers must ensure the policy is inactive by now, to avoid any + * races with show()/store() callbacks. + */ + if (unlikely(!policy_is_inactive(policy))) + pr_warn("%s: Freeing active policy\n", __func__); + + /* Remove policy from list */ write_lock_irqsave(&cpufreq_driver_lock, flags); - for_each_cpu(j, policy->cpus) - per_cpu(cpufreq_cpu_data, j) = NULL; + list_del(&policy->policy_list); + + for_each_cpu(cpu, policy->related_cpus) + per_cpu(cpufreq_cpu_data, cpu) = NULL; write_unlock_irqrestore(&cpufreq_driver_lock, flags); - kobject_put(&policy->kobj); - wait_for_completion(&policy->kobj_unregister); + freq_qos_remove_notifier(&policy->constraints, FREQ_QOS_MAX, + &policy->nb_max); + freq_qos_remove_notifier(&policy->constraints, FREQ_QOS_MIN, + &policy->nb_min); + + /* Cancel any pending policy->update work before freeing the policy. */ + cancel_work_sync(&policy->update); -err_set_policy_cpu: - per_cpu(cpufreq_policy_cpu, cpu) = -1; + if (policy->max_freq_req) { + /* + * Remove max_freq_req after sending CPUFREQ_REMOVE_POLICY + * notification, since CPUFREQ_CREATE_POLICY notification was + * sent after adding max_freq_req earlier. + */ + blocking_notifier_call_chain(&cpufreq_policy_notifier_list, + CPUFREQ_REMOVE_POLICY, policy); + freq_qos_remove_request(policy->max_freq_req); + } + + freq_qos_remove_request(policy->min_freq_req); + kfree(policy->min_freq_req); + + cpufreq_policy_put_kobj(policy); + free_cpumask_var(policy->real_cpus); free_cpumask_var(policy->related_cpus); -err_free_cpumask: free_cpumask_var(policy->cpus); -err_free_policy: kfree(policy); -nomem_out: - module_put(cpufreq_driver->owner); -module_out: - return ret; } -static void update_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu) +static int cpufreq_policy_online(struct cpufreq_policy *policy, + unsigned int cpu, bool new_policy) { - int j; + unsigned long flags; + unsigned int j; + int ret; + + guard(cpufreq_policy_write)(policy); - policy->last_cpu = policy->cpu; policy->cpu = cpu; + policy->governor = NULL; - for_each_cpu(j, policy->cpus) - per_cpu(cpufreq_policy_cpu, j) = cpu; + if (!new_policy && cpufreq_driver->online) { + /* Recover policy->cpus using related_cpus */ + cpumask_copy(policy->cpus, policy->related_cpus); -#ifdef CONFIG_CPU_FREQ_TABLE - cpufreq_frequency_table_update_policy_cpu(policy); -#endif - blocking_notifier_call_chain(&cpufreq_policy_notifier_list, - CPUFREQ_UPDATE_POLICY_CPU, policy); + ret = cpufreq_driver->online(policy); + if (ret) { + pr_debug("%s: %d: initialization failed\n", __func__, + __LINE__); + goto out_exit_policy; + } + } else { + cpumask_copy(policy->cpus, cpumask_of(cpu)); + + /* + * Call driver. From then on the cpufreq must be able + * to accept all calls to ->verify and ->setpolicy for this CPU. + */ + ret = cpufreq_driver->init(policy); + if (ret) { + pr_debug("%s: %d: initialization failed\n", __func__, + __LINE__); + goto out_clear_policy; + } + + /* + * The initialization has succeeded and the policy is online. + * If there is a problem with its frequency table, take it + * offline and drop it. + */ + if (policy->freq_table_sorted != CPUFREQ_TABLE_SORTED_ASCENDING && + policy->freq_table_sorted != CPUFREQ_TABLE_SORTED_DESCENDING) { + ret = cpufreq_table_validate_and_sort(policy); + if (ret) + goto out_offline_policy; + } + + /* related_cpus should at least include policy->cpus. */ + cpumask_copy(policy->related_cpus, policy->cpus); + } + + /* + * affected cpus must always be the one, which are online. We aren't + * managing offline cpus here. + */ + cpumask_and(policy->cpus, policy->cpus, cpu_online_mask); + + if (new_policy) { + for_each_cpu(j, policy->related_cpus) { + per_cpu(cpufreq_cpu_data, j) = policy; + add_cpu_dev_symlink(policy, j, get_cpu_device(j)); + } + + policy->min_freq_req = kzalloc(2 * sizeof(*policy->min_freq_req), + GFP_KERNEL); + if (!policy->min_freq_req) { + ret = -ENOMEM; + goto out_destroy_policy; + } + + ret = freq_qos_add_request(&policy->constraints, + policy->min_freq_req, FREQ_QOS_MIN, + FREQ_QOS_MIN_DEFAULT_VALUE); + if (ret < 0) { + /* + * So we don't call freq_qos_remove_request() for an + * uninitialized request. + */ + kfree(policy->min_freq_req); + policy->min_freq_req = NULL; + goto out_destroy_policy; + } + + /* + * This must be initialized right here to avoid calling + * freq_qos_remove_request() on uninitialized request in case + * of errors. + */ + policy->max_freq_req = policy->min_freq_req + 1; + + ret = freq_qos_add_request(&policy->constraints, + policy->max_freq_req, FREQ_QOS_MAX, + FREQ_QOS_MAX_DEFAULT_VALUE); + if (ret < 0) { + policy->max_freq_req = NULL; + goto out_destroy_policy; + } + + blocking_notifier_call_chain(&cpufreq_policy_notifier_list, + CPUFREQ_CREATE_POLICY, policy); + } else { + ret = freq_qos_update_request(policy->max_freq_req, policy->max); + if (ret < 0) + goto out_destroy_policy; + } + + if (cpufreq_driver->get && has_target()) { + policy->cur = cpufreq_driver->get(policy->cpu); + if (!policy->cur) { + ret = -EIO; + pr_err("%s: ->get() failed\n", __func__); + goto out_destroy_policy; + } + } + + /* + * Sometimes boot loaders set CPU frequency to a value outside of + * frequency table present with cpufreq core. In such cases CPU might be + * unstable if it has to run on that frequency for long duration of time + * and so its better to set it to a frequency which is specified in + * freq-table. This also makes cpufreq stats inconsistent as + * cpufreq-stats would fail to register because current frequency of CPU + * isn't found in freq-table. + * + * Because we don't want this change to effect boot process badly, we go + * for the next freq which is >= policy->cur ('cur' must be set by now, + * otherwise we will end up setting freq to lowest of the table as 'cur' + * is initialized to zero). + * + * We are passing target-freq as "policy->cur - 1" otherwise + * __cpufreq_driver_target() would simply fail, as policy->cur will be + * equal to target-freq. + */ + if ((cpufreq_driver->flags & CPUFREQ_NEED_INITIAL_FREQ_CHECK) + && has_target()) { + unsigned int old_freq = policy->cur; + + /* Are we running at unknown frequency ? */ + ret = cpufreq_frequency_table_get_index(policy, old_freq); + if (ret == -EINVAL) { + ret = __cpufreq_driver_target(policy, old_freq - 1, + CPUFREQ_RELATION_L); + + /* + * Reaching here after boot in a few seconds may not + * mean that system will remain stable at "unknown" + * frequency for longer duration. Hence, a BUG_ON(). + */ + BUG_ON(ret); + pr_info("%s: CPU%d: Running at unlisted initial frequency: %u kHz, changing to: %u kHz\n", + __func__, policy->cpu, old_freq, policy->cur); + } + } + + if (new_policy) { + ret = cpufreq_add_dev_interface(policy); + if (ret) + goto out_destroy_policy; + + cpufreq_stats_create_table(policy); + + write_lock_irqsave(&cpufreq_driver_lock, flags); + list_add(&policy->policy_list, &cpufreq_policy_list); + write_unlock_irqrestore(&cpufreq_driver_lock, flags); + + /* + * Register with the energy model before + * em_rebuild_sched_domains() is called, which will result + * in rebuilding of the sched domains, which should only be done + * once the energy model is properly initialized for the policy + * first. + * + * Also, this should be called before the policy is registered + * with cooling framework. + */ + if (cpufreq_driver->register_em) + cpufreq_driver->register_em(policy); + } + + ret = cpufreq_init_policy(policy); + if (ret) { + pr_err("%s: Failed to initialize policy for cpu: %d (%d)\n", + __func__, cpu, ret); + goto out_destroy_policy; + } + + return 0; + +out_destroy_policy: + for_each_cpu(j, policy->real_cpus) + remove_cpu_dev_symlink(policy, j, get_cpu_device(j)); + +out_offline_policy: + if (cpufreq_driver->offline) + cpufreq_driver->offline(policy); + +out_exit_policy: + if (cpufreq_driver->exit) + cpufreq_driver->exit(policy); + +out_clear_policy: + cpumask_clear(policy->cpus); + + return ret; } -/** - * __cpufreq_remove_dev - remove a CPU device - * - * Removes the cpufreq interface for a CPU device. - * Caller should already have policy_rwsem in write mode for this CPU. - * This routine frees the rwsem before returning. - */ -static int __cpufreq_remove_dev(struct device *dev, - struct subsys_interface *sif) +static int cpufreq_online(unsigned int cpu) { - unsigned int cpu = dev->id, ret, cpus; - unsigned long flags; - struct cpufreq_policy *data; - struct kobject *kobj; - struct completion *cmp; - struct device *cpu_dev; + struct cpufreq_policy *policy; + bool new_policy; + int ret; - pr_debug("%s: unregistering CPU %u\n", __func__, cpu); + pr_debug("%s: bringing CPU%u online\n", __func__, cpu); - write_lock_irqsave(&cpufreq_driver_lock, flags); + /* Check if this CPU already has a policy to manage it */ + policy = per_cpu(cpufreq_cpu_data, cpu); + if (policy) { + WARN_ON(!cpumask_test_cpu(cpu, policy->related_cpus)); + if (!policy_is_inactive(policy)) + return cpufreq_add_policy_cpu(policy, cpu); - data = per_cpu(cpufreq_cpu_data, cpu); - per_cpu(cpufreq_cpu_data, cpu) = NULL; + /* This is the only online CPU for the policy. Start over. */ + new_policy = false; + } else { + new_policy = true; + policy = cpufreq_policy_alloc(cpu); + if (!policy) + return -ENOMEM; + } - write_unlock_irqrestore(&cpufreq_driver_lock, flags); + ret = cpufreq_policy_online(policy, cpu, new_policy); + if (ret) { + cpufreq_policy_free(policy); + return ret; + } - if (!data) { - pr_debug("%s: No cpu_data found\n", __func__); - return -EINVAL; + kobject_uevent(&policy->kobj, KOBJ_ADD); + + /* Callback for handling stuff after policy is ready */ + if (cpufreq_driver->ready) + cpufreq_driver->ready(policy); + + /* Register cpufreq cooling only for a new policy */ + if (new_policy && cpufreq_thermal_control_enabled(cpufreq_driver)) + policy->cdev = of_cpufreq_cooling_register(policy); + + /* + * Let the per-policy boost flag mirror the cpufreq_driver boost during + * initialization for a new policy. For an existing policy, maintain the + * previous boost value unless global boost is disabled. + */ + if (cpufreq_driver->set_boost && policy->boost_supported && + (new_policy || !cpufreq_boost_enabled())) { + ret = policy_set_boost(policy, cpufreq_boost_enabled()); + if (ret) { + /* If the set_boost fails, the online operation is not affected */ + pr_info("%s: CPU%d: Cannot %s BOOST\n", __func__, policy->cpu, + str_enable_disable(cpufreq_boost_enabled())); + } } - if (cpufreq_driver->target) - __cpufreq_governor(data, CPUFREQ_GOV_STOP); + pr_debug("initialization complete\n"); -#ifdef CONFIG_HOTPLUG_CPU - if (!cpufreq_driver->setpolicy) - strncpy(per_cpu(cpufreq_cpu_governor, cpu), - data->governor->name, CPUFREQ_NAME_LEN); -#endif + return 0; +} - WARN_ON(lock_policy_rwsem_write(cpu)); - cpus = cpumask_weight(data->cpus); +/** + * cpufreq_add_dev - the cpufreq interface for a CPU device. + * @dev: CPU device. + * @sif: Subsystem interface structure pointer (not used) + */ +static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif) +{ + struct cpufreq_policy *policy; + unsigned cpu = dev->id; + int ret; - if (cpus > 1) - cpumask_clear_cpu(cpu, data->cpus); - unlock_policy_rwsem_write(cpu); + dev_dbg(dev, "%s: adding CPU%u\n", __func__, cpu); - if (cpu != data->cpu) { - sysfs_remove_link(&dev->kobj, "cpufreq"); - } else if (cpus > 1) { - /* first sibling now owns the new sysfs dir */ - cpu_dev = get_cpu_device(cpumask_first(data->cpus)); - sysfs_remove_link(&cpu_dev->kobj, "cpufreq"); - ret = kobject_move(&data->kobj, &cpu_dev->kobj); - if (ret) { - pr_err("%s: Failed to move kobj: %d", __func__, ret); + if (cpu_online(cpu)) { + ret = cpufreq_online(cpu); + if (ret) + return ret; + } - WARN_ON(lock_policy_rwsem_write(cpu)); - cpumask_set_cpu(cpu, data->cpus); + /* Create sysfs link on CPU registration */ + policy = per_cpu(cpufreq_cpu_data, cpu); + if (policy) + add_cpu_dev_symlink(policy, cpu, dev); + + return 0; +} - write_lock_irqsave(&cpufreq_driver_lock, flags); - per_cpu(cpufreq_cpu_data, cpu) = data; - write_unlock_irqrestore(&cpufreq_driver_lock, flags); +static void __cpufreq_offline(unsigned int cpu, struct cpufreq_policy *policy) +{ + int ret; - unlock_policy_rwsem_write(cpu); + if (has_target()) + cpufreq_stop_governor(policy); - ret = sysfs_create_link(&cpu_dev->kobj, &data->kobj, - "cpufreq"); - return -EINVAL; + cpumask_clear_cpu(cpu, policy->cpus); + + if (!policy_is_inactive(policy)) { + /* Nominate a new CPU if necessary. */ + if (cpu == policy->cpu) + policy->cpu = cpumask_any(policy->cpus); + + /* Start the governor again for the active policy. */ + if (has_target()) { + ret = cpufreq_start_governor(policy); + if (ret) + pr_err("%s: Failed to start governor\n", __func__); } - WARN_ON(lock_policy_rwsem_write(cpu)); - update_policy_cpu(data, cpu_dev->id); - unlock_policy_rwsem_write(cpu); - pr_debug("%s: policy Kobject moved to cpu: %d from: %d\n", - __func__, cpu_dev->id, cpu); + return; } - if ((cpus == 1) && (cpufreq_driver->target)) - __cpufreq_governor(data, CPUFREQ_GOV_POLICY_EXIT); + if (has_target()) { + strscpy(policy->last_governor, policy->governor->name, + CPUFREQ_NAME_LEN); + cpufreq_exit_governor(policy); + } else { + policy->last_policy = policy->policy; + } - pr_debug("%s: removing link, cpu: %d\n", __func__, cpu); - cpufreq_cpu_put(data); + /* + * Perform the ->offline() during light-weight tear-down, as + * that allows fast recovery when the CPU comes back. + */ + if (cpufreq_driver->offline) { + cpufreq_driver->offline(policy); + return; + } - /* If cpu is last user of policy, free policy */ - if (cpus == 1) { - lock_policy_rwsem_read(cpu); - kobj = &data->kobj; - cmp = &data->kobj_unregister; - unlock_policy_rwsem_read(cpu); - kobject_put(kobj); + if (cpufreq_driver->exit) + cpufreq_driver->exit(policy); - /* we need to make sure that the underlying kobj is actually - * not referenced anymore by anybody before we proceed with - * unloading. - */ - pr_debug("waiting for dropping of refcount\n"); - wait_for_completion(cmp); - pr_debug("wait complete\n"); + policy->freq_table = NULL; +} - if (cpufreq_driver->exit) - cpufreq_driver->exit(data); +static int cpufreq_offline(unsigned int cpu) +{ + struct cpufreq_policy *policy; + + pr_debug("%s: unregistering CPU %u\n", __func__, cpu); - free_cpumask_var(data->related_cpus); - free_cpumask_var(data->cpus); - kfree(data); - } else if (cpufreq_driver->target) { - __cpufreq_governor(data, CPUFREQ_GOV_START); - __cpufreq_governor(data, CPUFREQ_GOV_LIMITS); + policy = cpufreq_cpu_get_raw(cpu); + if (!policy) { + pr_debug("%s: No cpu_data found\n", __func__); + return 0; } - per_cpu(cpufreq_policy_cpu, cpu) = -1; + guard(cpufreq_policy_write)(policy); + + __cpufreq_offline(cpu, policy); + return 0; } -static int cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif) +/* + * cpufreq_remove_dev - remove a CPU device + * + * Removes the cpufreq interface for a CPU device. + */ +static void cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif) { unsigned int cpu = dev->id; - int retval; + struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu); - if (cpu_is_offline(cpu)) - return 0; + if (!policy) + return; - retval = __cpufreq_remove_dev(dev, sif); - return retval; -} + scoped_guard(cpufreq_policy_write, policy) { + if (cpu_online(cpu)) + __cpufreq_offline(cpu, policy); -static void handle_update(struct work_struct *work) -{ - struct cpufreq_policy *policy = - container_of(work, struct cpufreq_policy, update); - unsigned int cpu = policy->cpu; - pr_debug("handle_update for cpu %u called\n", cpu); - cpufreq_update_policy(cpu); + remove_cpu_dev_symlink(policy, cpu, dev); + + if (!cpumask_empty(policy->real_cpus)) + return; + + /* + * Unregister cpufreq cooling once all the CPUs of the policy + * are removed. + */ + if (cpufreq_thermal_control_enabled(cpufreq_driver)) { + cpufreq_cooling_unregister(policy->cdev); + policy->cdev = NULL; + } + + /* We did light-weight exit earlier, do full tear down now */ + if (cpufreq_driver->offline && cpufreq_driver->exit) + cpufreq_driver->exit(policy); + } + + cpufreq_policy_free(policy); } /** - * cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're - * in deep trouble. - * @cpu: cpu number - * @old_freq: CPU frequency the kernel thinks the CPU runs at - * @new_freq: CPU frequency the CPU actually runs at + * cpufreq_out_of_sync - Fix up actual and saved CPU frequency difference. + * @policy: Policy managing CPUs. + * @new_freq: New CPU frequency. * - * We adjust to current frequency first, and need to clean up later. - * So either call to cpufreq_update_policy() or schedule handle_update()). + * Adjust to the current frequency first and clean up later by either calling + * cpufreq_update_policy(), or scheduling handle_update(). */ -static void cpufreq_out_of_sync(unsigned int cpu, unsigned int old_freq, +static void cpufreq_out_of_sync(struct cpufreq_policy *policy, unsigned int new_freq) { - struct cpufreq_policy *policy; struct cpufreq_freqs freqs; - unsigned long flags; - pr_debug("Warning: CPU frequency out of sync: cpufreq and timing " - "core thinks of %u, is %u kHz.\n", old_freq, new_freq); + pr_debug("Warning: CPU frequency out of sync: cpufreq and timing core thinks of %u, is %u kHz\n", + policy->cur, new_freq); - freqs.old = old_freq; + freqs.old = policy->cur; freqs.new = new_freq; - read_lock_irqsave(&cpufreq_driver_lock, flags); - policy = per_cpu(cpufreq_cpu_data, cpu); - read_unlock_irqrestore(&cpufreq_driver_lock, flags); + cpufreq_freq_transition_begin(policy, &freqs); + cpufreq_freq_transition_end(policy, &freqs, 0); +} + +static unsigned int cpufreq_verify_current_freq(struct cpufreq_policy *policy, bool update) +{ + unsigned int new_freq; + + if (!cpufreq_driver->get) + return 0; - cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE); - cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); + new_freq = cpufreq_driver->get(policy->cpu); + if (!new_freq) + return 0; + + /* + * If fast frequency switching is used with the given policy, the check + * against policy->cur is pointless, so skip it in that case. + */ + if (policy->fast_switch_enabled || !has_target()) + return new_freq; + + if (policy->cur != new_freq) { + /* + * For some platforms, the frequency returned by hardware may be + * slightly different from what is provided in the frequency + * table, for example hardware may return 499 MHz instead of 500 + * MHz. In such cases it is better to avoid getting into + * unnecessary frequency updates. + */ + if (abs(policy->cur - new_freq) < KHZ_PER_MHZ) + return policy->cur; + + cpufreq_out_of_sync(policy, new_freq); + if (update) + schedule_work(&policy->update); + } + + return new_freq; } /** @@ -1275,19 +1848,25 @@ static void cpufreq_out_of_sync(unsigned int cpu, unsigned int old_freq, */ unsigned int cpufreq_quick_get(unsigned int cpu) { - struct cpufreq_policy *policy; - unsigned int ret_freq = 0; + unsigned long flags; - if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get) - return cpufreq_driver->get(cpu); + read_lock_irqsave(&cpufreq_driver_lock, flags); - policy = cpufreq_cpu_get(cpu); - if (policy) { - ret_freq = policy->cur; - cpufreq_cpu_put(policy); + if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get) { + unsigned int ret_freq = cpufreq_driver->get(cpu); + + read_unlock_irqrestore(&cpufreq_driver_lock, flags); + + return ret_freq; } - return ret_freq; + read_unlock_irqrestore(&cpufreq_driver_lock, flags); + + struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu); + if (policy) + return policy->cur; + + return 0; } EXPORT_SYMBOL(cpufreq_quick_get); @@ -1299,39 +1878,36 @@ EXPORT_SYMBOL(cpufreq_quick_get); */ unsigned int cpufreq_quick_get_max(unsigned int cpu) { - struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); - unsigned int ret_freq = 0; + struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu); + if (policy) + return policy->max; - if (policy) { - ret_freq = policy->max; - cpufreq_cpu_put(policy); - } - - return ret_freq; + return 0; } EXPORT_SYMBOL(cpufreq_quick_get_max); -static unsigned int __cpufreq_get(unsigned int cpu) +/** + * cpufreq_get_hw_max_freq - get the max hardware frequency of the CPU + * @cpu: CPU number + * + * The default return value is the max_freq field of cpuinfo. + */ +__weak unsigned int cpufreq_get_hw_max_freq(unsigned int cpu) { - struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu); - unsigned int ret_freq = 0; + struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu); + if (policy) + return policy->cpuinfo.max_freq; - if (!cpufreq_driver->get) - return ret_freq; - - ret_freq = cpufreq_driver->get(cpu); + return 0; +} +EXPORT_SYMBOL(cpufreq_get_hw_max_freq); - if (ret_freq && policy->cur && - !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) { - /* verify no discrepancy between actual and - saved value exists */ - if (unlikely(ret_freq != policy->cur)) { - cpufreq_out_of_sync(cpu, policy->cur, ret_freq); - schedule_work(&policy->update); - } - } +static unsigned int __cpufreq_get(struct cpufreq_policy *policy) +{ + if (unlikely(policy_is_inactive(policy))) + return 0; - return ret_freq; + return cpufreq_verify_current_freq(policy, true); } /** @@ -1342,23 +1918,13 @@ static unsigned int __cpufreq_get(unsigned int cpu) */ unsigned int cpufreq_get(unsigned int cpu) { - unsigned int ret_freq = 0; - struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); - + struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu); if (!policy) - goto out; - - if (unlikely(lock_policy_rwsem_read(cpu))) - goto out_policy; - - ret_freq = __cpufreq_get(cpu); + return 0; - unlock_policy_rwsem_read(cpu); + guard(cpufreq_policy_read)(policy); -out_policy: - cpufreq_cpu_put(policy); -out: - return ret_freq; + return __cpufreq_get(policy); } EXPORT_SYMBOL(cpufreq_get); @@ -1369,89 +1935,125 @@ static struct subsys_interface cpufreq_interface = { .remove_dev = cpufreq_remove_dev, }; +/* + * In case platform wants some specific frequency to be configured + * during suspend.. + */ +int cpufreq_generic_suspend(struct cpufreq_policy *policy) +{ + int ret; + + if (!policy->suspend_freq) { + pr_debug("%s: suspend_freq not defined\n", __func__); + return 0; + } + + pr_debug("%s: Setting suspend-freq: %u\n", __func__, + policy->suspend_freq); + + ret = __cpufreq_driver_target(policy, policy->suspend_freq, + CPUFREQ_RELATION_H); + if (ret) + pr_err("%s: unable to set suspend-freq: %u. err: %d\n", + __func__, policy->suspend_freq, ret); + + return ret; +} +EXPORT_SYMBOL(cpufreq_generic_suspend); + /** - * cpufreq_bp_suspend - Prepare the boot CPU for system suspend. + * cpufreq_suspend() - Suspend CPUFreq governors. * - * This function is only executed for the boot processor. The other CPUs - * have been put offline by means of CPU hotplug. + * Called during system wide Suspend/Hibernate cycles for suspending governors + * as some platforms can't change frequency after this point in suspend cycle. + * Because some of the devices (like: i2c, regulators, etc) they use for + * changing frequency are suspended quickly after this point. */ -static int cpufreq_bp_suspend(void) +void cpufreq_suspend(void) { - int ret = 0; + struct cpufreq_policy *policy; - int cpu = smp_processor_id(); - struct cpufreq_policy *cpu_policy; + if (!cpufreq_driver) + return; - pr_debug("suspending cpu %u\n", cpu); + if (!has_target() && !cpufreq_driver->suspend) + goto suspend; - /* If there's no policy for the boot CPU, we have nothing to do. */ - cpu_policy = cpufreq_cpu_get(cpu); - if (!cpu_policy) - return 0; + pr_debug("%s: Suspending Governors\n", __func__); - if (cpufreq_driver->suspend) { - ret = cpufreq_driver->suspend(cpu_policy); - if (ret) - printk(KERN_ERR "cpufreq: suspend failed in ->suspend " - "step on CPU %u\n", cpu_policy->cpu); + for_each_active_policy(policy) { + if (has_target()) { + scoped_guard(cpufreq_policy_write, policy) { + cpufreq_stop_governor(policy); + } + } + + if (cpufreq_driver->suspend && cpufreq_driver->suspend(policy)) + pr_err("%s: Failed to suspend driver: %s\n", __func__, + cpufreq_driver->name); } - cpufreq_cpu_put(cpu_policy); - return ret; +suspend: + cpufreq_suspended = true; } /** - * cpufreq_bp_resume - Restore proper frequency handling of the boot CPU. - * - * 1.) resume CPUfreq hardware support (cpufreq_driver->resume()) - * 2.) schedule call cpufreq_update_policy() ASAP as interrupts are - * restored. It will verify that the current freq is in sync with - * what we believe it to be. This is a bit later than when it - * should be, but nonethteless it's better than calling - * cpufreq_driver->get() here which might re-enable interrupts... + * cpufreq_resume() - Resume CPUFreq governors. * - * This function is only executed for the boot CPU. The other CPUs have not - * been turned on yet. + * Called during system wide Suspend/Hibernate cycle for resuming governors that + * are suspended with cpufreq_suspend(). */ -static void cpufreq_bp_resume(void) +void cpufreq_resume(void) { - int ret = 0; + struct cpufreq_policy *policy; + int ret; - int cpu = smp_processor_id(); - struct cpufreq_policy *cpu_policy; + if (!cpufreq_driver) + return; - pr_debug("resuming cpu %u\n", cpu); + if (unlikely(!cpufreq_suspended)) + return; + + cpufreq_suspended = false; - /* If there's no policy for the boot CPU, we have nothing to do. */ - cpu_policy = cpufreq_cpu_get(cpu); - if (!cpu_policy) + if (!has_target() && !cpufreq_driver->resume) return; - if (cpufreq_driver->resume) { - ret = cpufreq_driver->resume(cpu_policy); - if (ret) { - printk(KERN_ERR "cpufreq: resume failed in ->resume " - "step on CPU %u\n", cpu_policy->cpu); - goto fail; - } - } + pr_debug("%s: Resuming Governors\n", __func__); - schedule_work(&cpu_policy->update); + for_each_active_policy(policy) { + if (cpufreq_driver->resume && cpufreq_driver->resume(policy)) { + pr_err("%s: Failed to resume driver: %s\n", __func__, + cpufreq_driver->name); + } else if (has_target()) { + scoped_guard(cpufreq_policy_write, policy) { + ret = cpufreq_start_governor(policy); + } -fail: - cpufreq_cpu_put(cpu_policy); + if (ret) + pr_err("%s: Failed to start governor for CPU%u's policy\n", + __func__, policy->cpu); + } + } } -static struct syscore_ops cpufreq_syscore_ops = { - .suspend = cpufreq_bp_suspend, - .resume = cpufreq_bp_resume, -}; +/** + * cpufreq_driver_test_flags - Test cpufreq driver's flags against given ones. + * @flags: Flags to test against the current cpufreq driver's flags. + * + * Assumes that the driver is there, so callers must ensure that this is the + * case. + */ +bool cpufreq_driver_test_flags(u16 flags) +{ + return !!(cpufreq_driver->flags & flags); +} /** - * cpufreq_get_current_driver - return current driver's name + * cpufreq_get_current_driver - Return the current driver's name. * - * Return the name string of the currently loaded cpufreq driver - * or NULL, if none. + * Return the name string of the currently registered cpufreq driver or NULL if + * none. */ const char *cpufreq_get_current_driver(void) { @@ -1462,22 +2064,36 @@ const char *cpufreq_get_current_driver(void) } EXPORT_SYMBOL_GPL(cpufreq_get_current_driver); +/** + * cpufreq_get_driver_data - Return current driver data. + * + * Return the private data of the currently registered cpufreq driver, or NULL + * if no cpufreq driver has been registered. + */ +void *cpufreq_get_driver_data(void) +{ + if (cpufreq_driver) + return cpufreq_driver->driver_data; + + return NULL; +} +EXPORT_SYMBOL_GPL(cpufreq_get_driver_data); + /********************************************************************* * NOTIFIER LISTS INTERFACE * *********************************************************************/ /** - * cpufreq_register_notifier - register a driver with cpufreq - * @nb: notifier function to register - * @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER + * cpufreq_register_notifier - Register a notifier with cpufreq. + * @nb: notifier function to register. + * @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER. * - * Add a driver to one of two lists: either a list of drivers that - * are notified about clock rate changes (once before and once after - * the transition), or a list of drivers that are notified about - * changes in cpufreq policy. + * Add a notifier to one of two lists: either a list of notifiers that run on + * clock rate changes (once before and once after every transition), or a list + * of notifiers that ron on cpufreq policy changes. * - * This function may sleep, and has the same return conditions as - * blocking_notifier_chain_register. + * This function may sleep and it has the same return values as + * blocking_notifier_chain_register(). */ int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list) { @@ -1486,12 +2102,20 @@ int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list) if (cpufreq_disabled()) return -EINVAL; - WARN_ON(!init_cpufreq_transition_notifier_list_called); - switch (list) { case CPUFREQ_TRANSITION_NOTIFIER: + mutex_lock(&cpufreq_fast_switch_lock); + + if (cpufreq_fast_switch_count > 0) { + mutex_unlock(&cpufreq_fast_switch_lock); + return -EBUSY; + } ret = srcu_notifier_chain_register( &cpufreq_transition_notifier_list, nb); + if (!ret) + cpufreq_fast_switch_count--; + + mutex_unlock(&cpufreq_fast_switch_lock); break; case CPUFREQ_POLICY_NOTIFIER: ret = blocking_notifier_chain_register( @@ -1506,14 +2130,14 @@ int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list) EXPORT_SYMBOL(cpufreq_register_notifier); /** - * cpufreq_unregister_notifier - unregister a driver with cpufreq - * @nb: notifier block to be unregistered - * @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER + * cpufreq_unregister_notifier - Unregister a notifier from cpufreq. + * @nb: notifier block to be unregistered. + * @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER. * - * Remove a driver from the CPU frequency notifier list. + * Remove a notifier from one of the cpufreq notifier lists. * - * This function may sleep, and has the same return conditions as - * blocking_notifier_chain_unregister. + * This function may sleep and it has the same return values as + * blocking_notifier_chain_unregister(). */ int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list) { @@ -1524,8 +2148,14 @@ int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list) switch (list) { case CPUFREQ_TRANSITION_NOTIFIER: + mutex_lock(&cpufreq_fast_switch_lock); + ret = srcu_notifier_chain_unregister( &cpufreq_transition_notifier_list, nb); + if (!ret && !WARN_ON(cpufreq_fast_switch_count >= 0)) + cpufreq_fast_switch_count++; + + mutex_unlock(&cpufreq_fast_switch_lock); break; case CPUFREQ_POLICY_NOTIFIER: ret = blocking_notifier_chain_unregister( @@ -1544,34 +2174,221 @@ EXPORT_SYMBOL(cpufreq_unregister_notifier); * GOVERNORS * *********************************************************************/ +/** + * cpufreq_driver_fast_switch - Carry out a fast CPU frequency switch. + * @policy: cpufreq policy to switch the frequency for. + * @target_freq: New frequency to set (may be approximate). + * + * Carry out a fast frequency switch without sleeping. + * + * The driver's ->fast_switch() callback invoked by this function must be + * suitable for being called from within RCU-sched read-side critical sections + * and it is expected to select the minimum available frequency greater than or + * equal to @target_freq (CPUFREQ_RELATION_L). + * + * This function must not be called if policy->fast_switch_enabled is unset. + * + * Governors calling this function must guarantee that it will never be invoked + * twice in parallel for the same policy and that it will never be called in + * parallel with either ->target() or ->target_index() for the same policy. + * + * Returns the actual frequency set for the CPU. + * + * If 0 is returned by the driver's ->fast_switch() callback to indicate an + * error condition, the hardware configuration must be preserved. + */ +unsigned int cpufreq_driver_fast_switch(struct cpufreq_policy *policy, + unsigned int target_freq) +{ + unsigned int freq; + int cpu; + + target_freq = clamp_val(target_freq, policy->min, policy->max); + freq = cpufreq_driver->fast_switch(policy, target_freq); + + if (!freq) + return 0; + + policy->cur = freq; + arch_set_freq_scale(policy->related_cpus, freq, + arch_scale_freq_ref(policy->cpu)); + cpufreq_stats_record_transition(policy, freq); + + if (trace_cpu_frequency_enabled()) { + for_each_cpu(cpu, policy->cpus) + trace_cpu_frequency(freq, cpu); + } + + return freq; +} +EXPORT_SYMBOL_GPL(cpufreq_driver_fast_switch); + +/** + * cpufreq_driver_adjust_perf - Adjust CPU performance level in one go. + * @cpu: Target CPU. + * @min_perf: Minimum (required) performance level (units of @capacity). + * @target_perf: Target (desired) performance level (units of @capacity). + * @capacity: Capacity of the target CPU. + * + * Carry out a fast performance level switch of @cpu without sleeping. + * + * The driver's ->adjust_perf() callback invoked by this function must be + * suitable for being called from within RCU-sched read-side critical sections + * and it is expected to select a suitable performance level equal to or above + * @min_perf and preferably equal to or below @target_perf. + * + * This function must not be called if policy->fast_switch_enabled is unset. + * + * Governors calling this function must guarantee that it will never be invoked + * twice in parallel for the same CPU and that it will never be called in + * parallel with either ->target() or ->target_index() or ->fast_switch() for + * the same CPU. + */ +void cpufreq_driver_adjust_perf(unsigned int cpu, + unsigned long min_perf, + unsigned long target_perf, + unsigned long capacity) +{ + cpufreq_driver->adjust_perf(cpu, min_perf, target_perf, capacity); +} + +/** + * cpufreq_driver_has_adjust_perf - Check "direct fast switch" callback. + * + * Return 'true' if the ->adjust_perf callback is present for the + * current driver or 'false' otherwise. + */ +bool cpufreq_driver_has_adjust_perf(void) +{ + return !!cpufreq_driver->adjust_perf; +} + +/* Must set freqs->new to intermediate frequency */ +static int __target_intermediate(struct cpufreq_policy *policy, + struct cpufreq_freqs *freqs, int index) +{ + int ret; + + freqs->new = cpufreq_driver->get_intermediate(policy, index); + + /* We don't need to switch to intermediate freq */ + if (!freqs->new) + return 0; + + pr_debug("%s: cpu: %d, switching to intermediate freq: oldfreq: %u, intermediate freq: %u\n", + __func__, policy->cpu, freqs->old, freqs->new); + + cpufreq_freq_transition_begin(policy, freqs); + ret = cpufreq_driver->target_intermediate(policy, index); + cpufreq_freq_transition_end(policy, freqs, ret); + + if (ret) + pr_err("%s: Failed to change to intermediate frequency: %d\n", + __func__, ret); + + return ret; +} + +static int __target_index(struct cpufreq_policy *policy, int index) +{ + struct cpufreq_freqs freqs = {.old = policy->cur, .flags = 0}; + unsigned int restore_freq, intermediate_freq = 0; + unsigned int newfreq = policy->freq_table[index].frequency; + int retval = -EINVAL; + bool notify; + + if (newfreq == policy->cur) + return 0; + + /* Save last value to restore later on errors */ + restore_freq = policy->cur; + + notify = !(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION); + if (notify) { + /* Handle switching to intermediate frequency */ + if (cpufreq_driver->get_intermediate) { + retval = __target_intermediate(policy, &freqs, index); + if (retval) + return retval; + + intermediate_freq = freqs.new; + /* Set old freq to intermediate */ + if (intermediate_freq) + freqs.old = freqs.new; + } + + freqs.new = newfreq; + pr_debug("%s: cpu: %d, oldfreq: %u, new freq: %u\n", + __func__, policy->cpu, freqs.old, freqs.new); + + cpufreq_freq_transition_begin(policy, &freqs); + } + + retval = cpufreq_driver->target_index(policy, index); + if (retval) + pr_err("%s: Failed to change cpu frequency: %d\n", __func__, + retval); + + if (notify) { + cpufreq_freq_transition_end(policy, &freqs, retval); + + /* + * Failed after setting to intermediate freq? Driver should have + * reverted back to initial frequency and so should we. Check + * here for intermediate_freq instead of get_intermediate, in + * case we haven't switched to intermediate freq at all. + */ + if (unlikely(retval && intermediate_freq)) { + freqs.old = intermediate_freq; + freqs.new = restore_freq; + cpufreq_freq_transition_begin(policy, &freqs); + cpufreq_freq_transition_end(policy, &freqs, 0); + } + } + + return retval; +} + int __cpufreq_driver_target(struct cpufreq_policy *policy, unsigned int target_freq, unsigned int relation) { - int retval = -EINVAL; unsigned int old_target_freq = target_freq; if (cpufreq_disabled()) return -ENODEV; - if (policy->transition_ongoing) - return -EBUSY; - /* Make sure that target_freq is within supported range */ - if (target_freq > policy->max) - target_freq = policy->max; - if (target_freq < policy->min) - target_freq = policy->min; + target_freq = __resolve_freq(policy, target_freq, policy->min, + policy->max, relation); pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n", - policy->cpu, target_freq, relation, old_target_freq); + policy->cpu, target_freq, relation, old_target_freq); - if (target_freq == policy->cur) + /* + * This might look like a redundant call as we are checking it again + * after finding index. But it is left intentionally for cases where + * exactly same freq is called again and so we can save on few function + * calls. + */ + if (target_freq == policy->cur && + !(cpufreq_driver->flags & CPUFREQ_NEED_UPDATE_LIMITS)) return 0; - if (cpufreq_driver->target) - retval = cpufreq_driver->target(policy, target_freq, relation); + if (cpufreq_driver->target) { + /* + * If the driver hasn't setup a single inefficient frequency, + * it's unlikely it knows how to decode CPUFREQ_RELATION_E. + */ + if (!policy->efficiencies_available) + relation &= ~CPUFREQ_RELATION_E; - return retval; + return cpufreq_driver->target(policy, target_freq, relation); + } + + if (!cpufreq_driver->target_index) + return -EINVAL; + + return __target_index(policy, policy->cached_resolved_idx); } EXPORT_SYMBOL_GPL(__cpufreq_driver_target); @@ -1579,111 +2396,122 @@ int cpufreq_driver_target(struct cpufreq_policy *policy, unsigned int target_freq, unsigned int relation) { - int ret = -EINVAL; - - if (unlikely(lock_policy_rwsem_write(policy->cpu))) - goto fail; - - ret = __cpufreq_driver_target(policy, target_freq, relation); - - unlock_policy_rwsem_write(policy->cpu); + guard(cpufreq_policy_write)(policy); -fail: - return ret; + return __cpufreq_driver_target(policy, target_freq, relation); } EXPORT_SYMBOL_GPL(cpufreq_driver_target); -int __cpufreq_driver_getavg(struct cpufreq_policy *policy, unsigned int cpu) +__weak struct cpufreq_governor *cpufreq_fallback_governor(void) { - if (cpufreq_disabled()) - return 0; - - if (!cpufreq_driver->getavg) - return 0; - - return cpufreq_driver->getavg(policy, cpu); + return NULL; } -EXPORT_SYMBOL_GPL(__cpufreq_driver_getavg); - -/* - * when "event" is CPUFREQ_GOV_LIMITS - */ -static int __cpufreq_governor(struct cpufreq_policy *policy, - unsigned int event) +static int cpufreq_init_governor(struct cpufreq_policy *policy) { int ret; - /* Only must be defined when default governor is known to have latency - restrictions, like e.g. conservative or ondemand. - That this is the case is already ensured in Kconfig - */ -#ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE - struct cpufreq_governor *gov = &cpufreq_gov_performance; -#else - struct cpufreq_governor *gov = NULL; -#endif + /* Don't start any governor operations if we are entering suspend */ + if (cpufreq_suspended) + return 0; + /* + * Governor might not be initiated here if ACPI _PPC changed + * notification happened, so check it. + */ + if (!policy->governor) + return -EINVAL; - if (policy->governor->max_transition_latency && - policy->cpuinfo.transition_latency > - policy->governor->max_transition_latency) { - if (!gov) - return -EINVAL; - else { - printk(KERN_WARNING "%s governor failed, too long" - " transition latency of HW, fallback" - " to %s governor\n", - policy->governor->name, - gov->name); + /* Platform doesn't want dynamic frequency switching ? */ + if (policy->governor->flags & CPUFREQ_GOV_DYNAMIC_SWITCHING && + cpufreq_driver->flags & CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING) { + struct cpufreq_governor *gov = cpufreq_fallback_governor(); + + if (gov) { + pr_warn("Can't use %s governor as dynamic switching is disallowed. Fallback to %s governor\n", + policy->governor->name, gov->name); policy->governor = gov; + } else { + return -EINVAL; } } if (!try_module_get(policy->governor->owner)) return -EINVAL; - pr_debug("__cpufreq_governor for CPU %u, event %u\n", - policy->cpu, event); + pr_debug("%s: for CPU %u\n", __func__, policy->cpu); - mutex_lock(&cpufreq_governor_lock); - if ((!policy->governor_enabled && (event == CPUFREQ_GOV_STOP)) || - (policy->governor_enabled && (event == CPUFREQ_GOV_START))) { - mutex_unlock(&cpufreq_governor_lock); - return -EBUSY; + if (policy->governor->init) { + ret = policy->governor->init(policy); + if (ret) { + module_put(policy->governor->owner); + return ret; + } } - if (event == CPUFREQ_GOV_STOP) - policy->governor_enabled = false; - else if (event == CPUFREQ_GOV_START) - policy->governor_enabled = true; + policy->strict_target = !!(policy->governor->flags & CPUFREQ_GOV_STRICT_TARGET); - mutex_unlock(&cpufreq_governor_lock); + return 0; +} - ret = policy->governor->governor(policy, event); +static void cpufreq_exit_governor(struct cpufreq_policy *policy) +{ + if (cpufreq_suspended || !policy->governor) + return; - if (!ret) { - if (event == CPUFREQ_GOV_POLICY_INIT) - policy->governor->initialized++; - else if (event == CPUFREQ_GOV_POLICY_EXIT) - policy->governor->initialized--; - } else { - /* Restore original values */ - mutex_lock(&cpufreq_governor_lock); - if (event == CPUFREQ_GOV_STOP) - policy->governor_enabled = true; - else if (event == CPUFREQ_GOV_START) - policy->governor_enabled = false; - mutex_unlock(&cpufreq_governor_lock); - } - - /* we keep one module reference alive for - each CPU governed by this CPU */ - if ((event != CPUFREQ_GOV_START) || ret) - module_put(policy->governor->owner); - if ((event == CPUFREQ_GOV_STOP) && !ret) - module_put(policy->governor->owner); + pr_debug("%s: for CPU %u\n", __func__, policy->cpu); - return ret; + if (policy->governor->exit) + policy->governor->exit(policy); + + module_put(policy->governor->owner); +} + +int cpufreq_start_governor(struct cpufreq_policy *policy) +{ + int ret; + + if (cpufreq_suspended) + return 0; + + if (!policy->governor) + return -EINVAL; + + pr_debug("%s: for CPU %u\n", __func__, policy->cpu); + + cpufreq_verify_current_freq(policy, false); + + if (policy->governor->start) { + ret = policy->governor->start(policy); + if (ret) + return ret; + } + + if (policy->governor->limits) + policy->governor->limits(policy); + + return 0; +} + +void cpufreq_stop_governor(struct cpufreq_policy *policy) +{ + if (cpufreq_suspended || !policy->governor) + return; + + pr_debug("%s: for CPU %u\n", __func__, policy->cpu); + + if (policy->governor->stop) + policy->governor->stop(policy); +} + +static void cpufreq_governor_limits(struct cpufreq_policy *policy) +{ + if (cpufreq_suspended || !policy->governor) + return; + + pr_debug("%s: for CPU %u\n", __func__, policy->cpu); + + if (policy->governor->limits) + policy->governor->limits(policy); } int cpufreq_register_governor(struct cpufreq_governor *governor) @@ -1698,9 +2526,8 @@ int cpufreq_register_governor(struct cpufreq_governor *governor) mutex_lock(&cpufreq_governor_mutex); - governor->initialized = 0; err = -EBUSY; - if (__find_governor(governor->name) == NULL) { + if (!find_governor(governor->name)) { err = 0; list_add(&governor->governor_list, &cpufreq_governor_list); } @@ -1712,9 +2539,8 @@ EXPORT_SYMBOL_GPL(cpufreq_register_governor); void cpufreq_unregister_governor(struct cpufreq_governor *governor) { -#ifdef CONFIG_HOTPLUG_CPU - int cpu; -#endif + struct cpufreq_policy *policy; + unsigned long flags; if (!governor) return; @@ -1722,19 +2548,19 @@ void cpufreq_unregister_governor(struct cpufreq_governor *governor) if (cpufreq_disabled()) return; -#ifdef CONFIG_HOTPLUG_CPU - for_each_present_cpu(cpu) { - if (cpu_online(cpu)) - continue; - if (!strcmp(per_cpu(cpufreq_cpu_governor, cpu), governor->name)) - strcpy(per_cpu(cpufreq_cpu_governor, cpu), "\0"); + /* clear last_governor for all inactive policies */ + read_lock_irqsave(&cpufreq_driver_lock, flags); + for_each_inactive_policy(policy) { + if (!strcmp(policy->last_governor, governor->name)) { + policy->governor = NULL; + policy->last_governor[0] = '\0'; + } } -#endif + read_unlock_irqrestore(&cpufreq_driver_lock, flags); mutex_lock(&cpufreq_governor_mutex); list_del(&governor->governor_list); mutex_unlock(&cpufreq_governor_mutex); - return; } EXPORT_SYMBOL_GPL(cpufreq_unregister_governor); @@ -1743,226 +2569,324 @@ EXPORT_SYMBOL_GPL(cpufreq_unregister_governor); * POLICY INTERFACE * *********************************************************************/ +DEFINE_PER_CPU(unsigned long, cpufreq_pressure); + /** - * cpufreq_get_policy - get the current cpufreq_policy - * @policy: struct cpufreq_policy into which the current cpufreq_policy - * is written + * cpufreq_update_pressure() - Update cpufreq pressure for CPUs + * @policy: cpufreq policy of the CPUs. * - * Reads the current cpufreq policy. + * Update the value of cpufreq pressure for all @cpus in the policy. */ -int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu) +static void cpufreq_update_pressure(struct cpufreq_policy *policy) { - struct cpufreq_policy *cpu_policy; - if (!policy) - return -EINVAL; + unsigned long max_capacity, capped_freq, pressure; + u32 max_freq; + int cpu; - cpu_policy = cpufreq_cpu_get(cpu); - if (!cpu_policy) - return -EINVAL; + cpu = cpumask_first(policy->related_cpus); + max_freq = arch_scale_freq_ref(cpu); + capped_freq = policy->max; - memcpy(policy, cpu_policy, sizeof(struct cpufreq_policy)); + /* + * Handle properly the boost frequencies, which should simply clean + * the cpufreq pressure value. + */ + if (max_freq <= capped_freq) { + pressure = 0; + } else { + max_capacity = arch_scale_cpu_capacity(cpu); + pressure = max_capacity - + mult_frac(max_capacity, capped_freq, max_freq); + } - cpufreq_cpu_put(cpu_policy); - return 0; + for_each_cpu(cpu, policy->related_cpus) + WRITE_ONCE(per_cpu(cpufreq_pressure, cpu), pressure); } -EXPORT_SYMBOL(cpufreq_get_policy); -/* - * data : current policy. - * policy : policy to be set. +/** + * cpufreq_set_policy - Modify cpufreq policy parameters. + * @policy: Policy object to modify. + * @new_gov: Policy governor pointer. + * @new_pol: Policy value (for drivers with built-in governors). + * + * Invoke the cpufreq driver's ->verify() callback to sanity-check the frequency + * limits to be set for the policy, update @policy with the verified limits + * values and either invoke the driver's ->setpolicy() callback (if present) or + * carry out a governor update for @policy. That is, run the current governor's + * ->limits() callback (if @new_gov points to the same object as the one in + * @policy) or replace the governor for @policy with @new_gov. + * + * The cpuinfo part of @policy is not updated by this function. */ -static int __cpufreq_set_policy(struct cpufreq_policy *data, - struct cpufreq_policy *policy) +static int cpufreq_set_policy(struct cpufreq_policy *policy, + struct cpufreq_governor *new_gov, + unsigned int new_pol) { - int ret = 0, failed = 1; - - pr_debug("setting new policy for CPU %u: %u - %u kHz\n", policy->cpu, - policy->min, policy->max); + struct cpufreq_policy_data new_data; + struct cpufreq_governor *old_gov; + int ret; - memcpy(&policy->cpuinfo, &data->cpuinfo, - sizeof(struct cpufreq_cpuinfo)); + memcpy(&new_data.cpuinfo, &policy->cpuinfo, sizeof(policy->cpuinfo)); + new_data.freq_table = policy->freq_table; + new_data.cpu = policy->cpu; + /* + * PM QoS framework collects all the requests from users and provide us + * the final aggregated value here. + */ + new_data.min = freq_qos_read_value(&policy->constraints, FREQ_QOS_MIN); + new_data.max = freq_qos_read_value(&policy->constraints, FREQ_QOS_MAX); - if (policy->min > data->max || policy->max < data->min) { - ret = -EINVAL; - goto error_out; - } + pr_debug("setting new policy for CPU %u: %u - %u kHz\n", + new_data.cpu, new_data.min, new_data.max); - /* verify the cpu speed can be set within this limit */ - ret = cpufreq_driver->verify(policy); + /* + * Verify that the CPU speed can be set within these limits and make sure + * that min <= max. + */ + ret = cpufreq_driver->verify(&new_data); if (ret) - goto error_out; - - /* adjust if necessary - all reasons */ - blocking_notifier_call_chain(&cpufreq_policy_notifier_list, - CPUFREQ_ADJUST, policy); - - /* adjust if necessary - hardware incompatibility*/ - blocking_notifier_call_chain(&cpufreq_policy_notifier_list, - CPUFREQ_INCOMPATIBLE, policy); + return ret; /* - * verify the cpu speed can be set within this limit, which might be - * different to the first one + * Resolve policy min/max to available frequencies. It ensures + * no frequency resolution will neither overshoot the requested maximum + * nor undershoot the requested minimum. + * + * Avoid storing intermediate values in policy->max or policy->min and + * compiler optimizations around them because they may be accessed + * concurrently by cpufreq_driver_resolve_freq() during the update. */ - ret = cpufreq_driver->verify(policy); - if (ret) - goto error_out; + WRITE_ONCE(policy->max, __resolve_freq(policy, new_data.max, + new_data.min, new_data.max, + CPUFREQ_RELATION_H)); + new_data.min = __resolve_freq(policy, new_data.min, new_data.min, + new_data.max, CPUFREQ_RELATION_L); + WRITE_ONCE(policy->min, new_data.min > policy->max ? policy->max : new_data.min); - /* notification of the new policy */ - blocking_notifier_call_chain(&cpufreq_policy_notifier_list, - CPUFREQ_NOTIFY, policy); + trace_cpu_frequency_limits(policy); - data->min = policy->min; - data->max = policy->max; + cpufreq_update_pressure(policy); + + policy->cached_target_freq = UINT_MAX; pr_debug("new min and max freqs are %u - %u kHz\n", - data->min, data->max); + policy->min, policy->max); if (cpufreq_driver->setpolicy) { - data->policy = policy->policy; + policy->policy = new_pol; pr_debug("setting range\n"); - ret = cpufreq_driver->setpolicy(policy); - } else { - if (policy->governor != data->governor) { - /* save old, working values */ - struct cpufreq_governor *old_gov = data->governor; - - pr_debug("governor switch\n"); - - /* end old governor */ - if (data->governor) { - __cpufreq_governor(data, CPUFREQ_GOV_STOP); - unlock_policy_rwsem_write(policy->cpu); - __cpufreq_governor(data, - CPUFREQ_GOV_POLICY_EXIT); - lock_policy_rwsem_write(policy->cpu); - } + return cpufreq_driver->setpolicy(policy); + } - /* start new governor */ - data->governor = policy->governor; - if (!__cpufreq_governor(data, CPUFREQ_GOV_POLICY_INIT)) { - if (!__cpufreq_governor(data, CPUFREQ_GOV_START)) { - failed = 0; - } else { - unlock_policy_rwsem_write(policy->cpu); - __cpufreq_governor(data, - CPUFREQ_GOV_POLICY_EXIT); - lock_policy_rwsem_write(policy->cpu); - } - } + if (new_gov == policy->governor) { + pr_debug("governor limits update\n"); + cpufreq_governor_limits(policy); + return 0; + } - if (failed) { - /* new governor failed, so re-start old one */ - pr_debug("starting governor %s failed\n", - data->governor->name); - if (old_gov) { - data->governor = old_gov; - __cpufreq_governor(data, - CPUFREQ_GOV_POLICY_INIT); - __cpufreq_governor(data, - CPUFREQ_GOV_START); - } - ret = -EINVAL; - goto error_out; - } - /* might be a policy change, too, so fall through */ + pr_debug("governor switch\n"); + + /* save old, working values */ + old_gov = policy->governor; + /* end old governor */ + if (old_gov) { + cpufreq_stop_governor(policy); + cpufreq_exit_governor(policy); + } + + /* start new governor */ + policy->governor = new_gov; + ret = cpufreq_init_governor(policy); + if (!ret) { + ret = cpufreq_start_governor(policy); + if (!ret) { + pr_debug("governor change\n"); + return 0; + } + cpufreq_exit_governor(policy); + } + + /* new governor failed, so re-start old one */ + pr_debug("starting governor %s failed\n", policy->governor->name); + if (old_gov) { + policy->governor = old_gov; + if (cpufreq_init_governor(policy)) { + policy->governor = NULL; + } else if (cpufreq_start_governor(policy)) { + cpufreq_exit_governor(policy); + policy->governor = NULL; } - pr_debug("governor: change or update limits\n"); - __cpufreq_governor(data, CPUFREQ_GOV_LIMITS); } -error_out: return ret; } +static void cpufreq_policy_refresh(struct cpufreq_policy *policy) +{ + guard(cpufreq_policy_write)(policy); + + /* + * BIOS might change freq behind our back + * -> ask driver for current freq and notify governors about a change + */ + if (cpufreq_driver->get && has_target() && + (cpufreq_suspended || WARN_ON(!cpufreq_verify_current_freq(policy, false)))) + return; + + refresh_frequency_limits(policy); +} + +/** + * cpufreq_update_policy - Re-evaluate an existing cpufreq policy. + * @cpu: CPU to re-evaluate the policy for. + * + * Update the current frequency for the cpufreq policy of @cpu and use + * cpufreq_set_policy() to re-apply the min and max limits, which triggers the + * evaluation of policy notifiers and the cpufreq driver's ->verify() callback + * for the policy in question, among other things. + */ +void cpufreq_update_policy(unsigned int cpu) +{ + struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu); + if (!policy) + return; + + cpufreq_policy_refresh(policy); +} +EXPORT_SYMBOL(cpufreq_update_policy); + /** - * cpufreq_update_policy - re-evaluate an existing cpufreq policy - * @cpu: CPU which shall be re-evaluated + * cpufreq_update_limits - Update policy limits for a given CPU. + * @cpu: CPU to update the policy limits for. * - * Useful for policy notifiers which have different necessities - * at different times. + * Invoke the driver's ->update_limits callback if present or call + * cpufreq_policy_refresh() for @cpu. */ -int cpufreq_update_policy(unsigned int cpu) +void cpufreq_update_limits(unsigned int cpu) +{ + struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu); + if (!policy) + return; + + if (cpufreq_driver->update_limits) + cpufreq_driver->update_limits(policy); + else + cpufreq_policy_refresh(policy); +} +EXPORT_SYMBOL_GPL(cpufreq_update_limits); + +/********************************************************************* + * BOOST * + *********************************************************************/ +int cpufreq_boost_set_sw(struct cpufreq_policy *policy, int state) { - struct cpufreq_policy *data = cpufreq_cpu_get(cpu); - struct cpufreq_policy policy; int ret; - if (!data) { - ret = -ENODEV; - goto no_policy; - } + if (!policy->freq_table) + return -ENXIO; - if (unlikely(lock_policy_rwsem_write(cpu))) { - ret = -EINVAL; - goto fail; + ret = cpufreq_frequency_table_cpuinfo(policy); + if (ret) { + pr_err("%s: Policy frequency update failed\n", __func__); + return ret; } - pr_debug("updating policy for CPU %u\n", cpu); - memcpy(&policy, data, sizeof(struct cpufreq_policy)); - policy.min = data->user_policy.min; - policy.max = data->user_policy.max; - policy.policy = data->user_policy.policy; - policy.governor = data->user_policy.governor; + ret = freq_qos_update_request(policy->max_freq_req, policy->max); + if (ret < 0) + return ret; + + return 0; +} +EXPORT_SYMBOL_GPL(cpufreq_boost_set_sw); + +static int cpufreq_boost_trigger_state(int state) +{ + struct cpufreq_policy *policy; + unsigned long flags; + int ret = 0; /* - * BIOS might change freq behind our back - * -> ask driver for current freq and notify governors about a change + * Don't compare 'cpufreq_driver->boost_enabled' with 'state' here to + * make sure all policies are in sync with global boost flag. */ - if (cpufreq_driver->get) { - policy.cur = cpufreq_driver->get(cpu); - if (!data->cur) { - pr_debug("Driver did not initialize current freq"); - data->cur = policy.cur; - } else { - if (data->cur != policy.cur && cpufreq_driver->target) - cpufreq_out_of_sync(cpu, data->cur, - policy.cur); - } + + write_lock_irqsave(&cpufreq_driver_lock, flags); + cpufreq_driver->boost_enabled = state; + write_unlock_irqrestore(&cpufreq_driver_lock, flags); + + cpus_read_lock(); + for_each_active_policy(policy) { + if (!policy->boost_supported) + continue; + + ret = policy_set_boost(policy, state); + if (ret) + goto err_reset_state; } + cpus_read_unlock(); - ret = __cpufreq_set_policy(data, &policy); + return 0; + +err_reset_state: + cpus_read_unlock(); + + write_lock_irqsave(&cpufreq_driver_lock, flags); + cpufreq_driver->boost_enabled = !state; + write_unlock_irqrestore(&cpufreq_driver_lock, flags); - unlock_policy_rwsem_write(cpu); + pr_err("%s: Cannot %s BOOST\n", + __func__, str_enable_disable(state)); -fail: - cpufreq_cpu_put(data); -no_policy: return ret; } -EXPORT_SYMBOL(cpufreq_update_policy); -static int __cpuinit cpufreq_cpu_callback(struct notifier_block *nfb, - unsigned long action, void *hcpu) +static bool cpufreq_boost_supported(void) { - unsigned int cpu = (unsigned long)hcpu; - struct device *dev; + return cpufreq_driver->set_boost; +} - dev = get_cpu_device(cpu); - if (dev) { - switch (action) { - case CPU_ONLINE: - cpufreq_add_dev(dev, NULL); - break; - case CPU_DOWN_PREPARE: - case CPU_UP_CANCELED_FROZEN: - __cpufreq_remove_dev(dev, NULL); - break; - case CPU_DOWN_FAILED: - cpufreq_add_dev(dev, NULL); - break; - } - } - return NOTIFY_OK; +static int create_boost_sysfs_file(void) +{ + int ret; + + ret = sysfs_create_file(cpufreq_global_kobject, &boost.attr); + if (ret) + pr_err("%s: cannot register global BOOST sysfs file\n", + __func__); + + return ret; } -static struct notifier_block __refdata cpufreq_cpu_notifier = { - .notifier_call = cpufreq_cpu_callback, -}; +static void remove_boost_sysfs_file(void) +{ + if (cpufreq_boost_supported()) + sysfs_remove_file(cpufreq_global_kobject, &boost.attr); +} + +bool cpufreq_boost_enabled(void) +{ + return cpufreq_driver->boost_enabled; +} +EXPORT_SYMBOL_GPL(cpufreq_boost_enabled); /********************************************************************* * REGISTER / UNREGISTER CPUFREQ DRIVER * *********************************************************************/ +static enum cpuhp_state hp_online; + +static int cpuhp_cpufreq_online(unsigned int cpu) +{ + cpufreq_online(cpu); + + return 0; +} + +static int cpuhp_cpufreq_offline(unsigned int cpu) +{ + cpufreq_offline(cpu); + + return 0; +} /** * cpufreq_register_driver - register a CPU Frequency driver @@ -1970,7 +2894,7 @@ static struct notifier_block __refdata cpufreq_cpu_notifier = { * submitted by the CPU Frequency driver. * * Registers a CPU Frequency driver to this core code. This code - * returns zero on success, -EBUSY when another driver got here first + * returns zero on success, -EEXIST when another driver got here first * (and isn't unregistered in the meantime). * */ @@ -1982,61 +2906,94 @@ int cpufreq_register_driver(struct cpufreq_driver *driver_data) if (cpufreq_disabled()) return -ENODEV; + /* + * The cpufreq core depends heavily on the availability of device + * structure, make sure they are available before proceeding further. + */ + if (!get_cpu_device(0)) + return -EPROBE_DEFER; + if (!driver_data || !driver_data->verify || !driver_data->init || - ((!driver_data->setpolicy) && (!driver_data->target))) + (driver_data->target_index && driver_data->target) || + (!!driver_data->setpolicy == (driver_data->target_index || driver_data->target)) || + (!driver_data->get_intermediate != !driver_data->target_intermediate) || + (!driver_data->online != !driver_data->offline) || + (driver_data->adjust_perf && !driver_data->fast_switch)) return -EINVAL; pr_debug("trying to register driver %s\n", driver_data->name); - if (driver_data->setpolicy) - driver_data->flags |= CPUFREQ_CONST_LOOPS; + /* Protect against concurrent CPU online/offline. */ + cpus_read_lock(); write_lock_irqsave(&cpufreq_driver_lock, flags); if (cpufreq_driver) { write_unlock_irqrestore(&cpufreq_driver_lock, flags); - return -EBUSY; + ret = -EEXIST; + goto out; } cpufreq_driver = driver_data; write_unlock_irqrestore(&cpufreq_driver_lock, flags); - ret = subsys_interface_register(&cpufreq_interface); - if (ret) - goto err_null_driver; + if (driver_data->setpolicy) + driver_data->flags |= CPUFREQ_CONST_LOOPS; - if (!(cpufreq_driver->flags & CPUFREQ_STICKY)) { - int i; - ret = -ENODEV; + if (cpufreq_boost_supported()) { + ret = create_boost_sysfs_file(); + if (ret) + goto err_null_driver; + } - /* check for at least one working CPU */ - for (i = 0; i < nr_cpu_ids; i++) - if (cpu_possible(i) && per_cpu(cpufreq_cpu_data, i)) { - ret = 0; - break; - } + /* + * Mark support for the scheduler's frequency invariance engine for + * drivers that implement target(), target_index() or fast_switch(). + */ + if (!cpufreq_driver->setpolicy) { + static_branch_enable_cpuslocked(&cpufreq_freq_invariance); + pr_debug("cpufreq: supports frequency invariance\n"); + } + ret = subsys_interface_register(&cpufreq_interface); + if (ret) + goto err_boost_unreg; + + if (unlikely(list_empty(&cpufreq_policy_list))) { /* if all ->init() calls failed, unregister */ - if (ret) { - pr_debug("no CPU initialized for driver %s\n", - driver_data->name); - goto err_if_unreg; - } + ret = -ENODEV; + pr_debug("%s: No CPU initialized for driver %s\n", __func__, + driver_data->name); + goto err_if_unreg; } - register_hotcpu_notifier(&cpufreq_cpu_notifier); + ret = cpuhp_setup_state_nocalls_cpuslocked(CPUHP_AP_ONLINE_DYN, + "cpufreq:online", + cpuhp_cpufreq_online, + cpuhp_cpufreq_offline); + if (ret < 0) + goto err_if_unreg; + hp_online = ret; + ret = 0; + pr_debug("driver %s up and running\n", driver_data->name); + goto out; - return 0; err_if_unreg: subsys_interface_unregister(&cpufreq_interface); +err_boost_unreg: + if (!cpufreq_driver->setpolicy) + static_branch_disable_cpuslocked(&cpufreq_freq_invariance); + remove_boost_sysfs_file(); err_null_driver: write_lock_irqsave(&cpufreq_driver_lock, flags); cpufreq_driver = NULL; write_unlock_irqrestore(&cpufreq_driver_lock, flags); +out: + cpus_read_unlock(); return ret; } EXPORT_SYMBOL_GPL(cpufreq_register_driver); -/** +/* * cpufreq_unregister_driver - unregister the current CPUFreq driver * * Unregister the current CPUFreq driver. Only call this if you have @@ -2044,42 +3001,79 @@ EXPORT_SYMBOL_GPL(cpufreq_register_driver); * Returns zero if successful, and -EINVAL if the cpufreq_driver is * currently not initialised. */ -int cpufreq_unregister_driver(struct cpufreq_driver *driver) +void cpufreq_unregister_driver(struct cpufreq_driver *driver) { unsigned long flags; - if (!cpufreq_driver || (driver != cpufreq_driver)) - return -EINVAL; + if (WARN_ON(!cpufreq_driver || (driver != cpufreq_driver))) + return; pr_debug("unregistering driver %s\n", driver->name); + /* Protect against concurrent cpu hotplug */ + cpus_read_lock(); subsys_interface_unregister(&cpufreq_interface); - unregister_hotcpu_notifier(&cpufreq_cpu_notifier); + remove_boost_sysfs_file(); + static_branch_disable_cpuslocked(&cpufreq_freq_invariance); + cpuhp_remove_state_nocalls_cpuslocked(hp_online); write_lock_irqsave(&cpufreq_driver_lock, flags); + cpufreq_driver = NULL; - write_unlock_irqrestore(&cpufreq_driver_lock, flags); - return 0; + write_unlock_irqrestore(&cpufreq_driver_lock, flags); + cpus_read_unlock(); } EXPORT_SYMBOL_GPL(cpufreq_unregister_driver); static int __init cpufreq_core_init(void) { - int cpu; + struct cpufreq_governor *gov = cpufreq_default_governor(); + struct device *dev_root; if (cpufreq_disabled()) return -ENODEV; - for_each_possible_cpu(cpu) { - per_cpu(cpufreq_policy_cpu, cpu) = -1; - init_rwsem(&per_cpu(cpu_policy_rwsem, cpu)); + dev_root = bus_get_dev_root(&cpu_subsys); + if (dev_root) { + cpufreq_global_kobject = kobject_create_and_add("cpufreq", &dev_root->kobj); + put_device(dev_root); } - - cpufreq_global_kobject = kobject_create(); BUG_ON(!cpufreq_global_kobject); - register_syscore_ops(&cpufreq_syscore_ops); + + if (!strlen(default_governor)) + strscpy(default_governor, gov->name, CPUFREQ_NAME_LEN); return 0; } + +static bool cpufreq_policy_is_good_for_eas(unsigned int cpu) +{ + struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu); + if (!policy) { + pr_debug("cpufreq policy not set for CPU: %d\n", cpu); + return false; + } + + return sugov_is_governor(policy); +} + +bool cpufreq_ready_for_eas(const struct cpumask *cpu_mask) +{ + unsigned int cpu; + + /* Do not attempt EAS if schedutil is not being used. */ + for_each_cpu(cpu, cpu_mask) { + if (!cpufreq_policy_is_good_for_eas(cpu)) { + pr_debug("rd %*pbl: schedutil is mandatory for EAS\n", + cpumask_pr_args(cpu_mask)); + return false; + } + } + + return true; +} + +module_param(off, int, 0444); +module_param_string(default_governor, default_governor, CPUFREQ_NAME_LEN, 0444); core_initcall(cpufreq_core_init); diff --git a/drivers/cpufreq/cpufreq_conservative.c b/drivers/cpufreq/cpufreq_conservative.c index 0ceb2eff5a7e..cce6a8d113e1 100644 --- a/drivers/cpufreq/cpufreq_conservative.c +++ b/drivers/cpufreq/cpufreq_conservative.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * drivers/cpufreq/cpufreq_conservative.c * @@ -5,27 +6,27 @@ * (C) 2003 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>. * Jun Nakajima <jun.nakajima@intel.com> * (C) 2009 Alexander Clouter <alex@digriz.org.uk> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ -#include <linux/cpufreq.h> -#include <linux/init.h> -#include <linux/kernel.h> -#include <linux/kernel_stat.h> -#include <linux/kobject.h> -#include <linux/module.h> -#include <linux/mutex.h> -#include <linux/notifier.h> -#include <linux/percpu-defs.h> #include <linux/slab.h> -#include <linux/sysfs.h> -#include <linux/types.h> - #include "cpufreq_governor.h" +struct cs_policy_dbs_info { + struct policy_dbs_info policy_dbs; + unsigned int down_skip; + unsigned int requested_freq; +}; + +static inline struct cs_policy_dbs_info *to_dbs_info(struct policy_dbs_info *policy_dbs) +{ + return container_of(policy_dbs, struct cs_policy_dbs_info, policy_dbs); +} + +struct cs_dbs_tuners { + unsigned int down_threshold; + unsigned int freq_step; +}; + /* Conservative governor macros */ #define DEF_FREQUENCY_UP_THRESHOLD (80) #define DEF_FREQUENCY_DOWN_THRESHOLD (20) @@ -33,18 +34,16 @@ #define DEF_SAMPLING_DOWN_FACTOR (1) #define MAX_SAMPLING_DOWN_FACTOR (10) -static DEFINE_PER_CPU(struct cs_cpu_dbs_info_s, cs_cpu_dbs_info); - -static inline unsigned int get_freq_target(struct cs_dbs_tuners *cs_tuners, - struct cpufreq_policy *policy) +static inline unsigned int get_freq_step(struct cs_dbs_tuners *cs_tuners, + struct cpufreq_policy *policy) { - unsigned int freq_target = (cs_tuners->freq_step * policy->max) / 100; + unsigned int freq_step = (cs_tuners->freq_step * policy->max) / 100; /* max freq cannot be less than 100. But who knows... */ - if (unlikely(freq_target == 0)) - freq_target = DEF_FREQUENCY_STEP; + if (unlikely(freq_step == 0)) + freq_step = DEF_FREQUENCY_STEP; - return freq_target; + return freq_step; } /* @@ -53,43 +52,74 @@ static inline unsigned int get_freq_target(struct cs_dbs_tuners *cs_tuners, * sampling_down_factor, we check, if current idle time is more than 80% * (default), then we try to decrease frequency * - * Any frequency increase takes it to the maximum frequency. Frequency reduction - * happens at minimum steps of 5% (default) of maximum frequency + * Frequency updates happen at minimum steps of 5% (default) of maximum + * frequency */ -static void cs_check_cpu(int cpu, unsigned int load) +static unsigned int cs_dbs_update(struct cpufreq_policy *policy) { - struct cs_cpu_dbs_info_s *dbs_info = &per_cpu(cs_cpu_dbs_info, cpu); - struct cpufreq_policy *policy = dbs_info->cdbs.cur_policy; - struct dbs_data *dbs_data = policy->governor_data; + struct policy_dbs_info *policy_dbs = policy->governor_data; + struct cs_policy_dbs_info *dbs_info = to_dbs_info(policy_dbs); + unsigned int requested_freq = dbs_info->requested_freq; + struct dbs_data *dbs_data = policy_dbs->dbs_data; struct cs_dbs_tuners *cs_tuners = dbs_data->tuners; + unsigned int load = dbs_update(policy); + unsigned int freq_step; /* * break out if we 'cannot' reduce the speed as the user might * want freq_step to be zero */ if (cs_tuners->freq_step == 0) - return; + goto out; + + /* + * If requested_freq is out of range, it is likely that the limits + * changed in the meantime, so fall back to current frequency in that + * case. + */ + if (requested_freq > policy->max || requested_freq < policy->min) { + requested_freq = policy->cur; + dbs_info->requested_freq = requested_freq; + } + + freq_step = get_freq_step(cs_tuners, policy); + + /* + * Decrease requested_freq one freq_step for each idle period that + * we didn't update the frequency. + */ + if (policy_dbs->idle_periods < UINT_MAX) { + unsigned int freq_steps = policy_dbs->idle_periods * freq_step; + + if (requested_freq > policy->min + freq_steps) + requested_freq -= freq_steps; + else + requested_freq = policy->min; + + policy_dbs->idle_periods = UINT_MAX; + } /* Check for frequency increase */ - if (load > cs_tuners->up_threshold) { + if (load > dbs_data->up_threshold) { dbs_info->down_skip = 0; /* if we are already at full speed then break out early */ - if (dbs_info->requested_freq == policy->max) - return; + if (requested_freq == policy->max) + goto out; - dbs_info->requested_freq += get_freq_target(cs_tuners, policy); - if (dbs_info->requested_freq > policy->max) - dbs_info->requested_freq = policy->max; + requested_freq += freq_step; + if (requested_freq > policy->max) + requested_freq = policy->max; - __cpufreq_driver_target(policy, dbs_info->requested_freq, - CPUFREQ_RELATION_H); - return; + __cpufreq_driver_target(policy, requested_freq, + CPUFREQ_RELATION_HE); + dbs_info->requested_freq = requested_freq; + goto out; } /* if sampling_down_factor is active break out early */ - if (++dbs_info->down_skip < cs_tuners->sampling_down_factor) - return; + if (++dbs_info->down_skip < dbs_data->sampling_down_factor) + goto out; dbs_info->down_skip = 0; /* Check for frequency decrease */ @@ -97,172 +127,109 @@ static void cs_check_cpu(int cpu, unsigned int load) /* * if we cannot reduce the frequency anymore, break out early */ - if (policy->cur == policy->min) - return; + if (requested_freq == policy->min) + goto out; - dbs_info->requested_freq -= get_freq_target(cs_tuners, policy); - if (dbs_info->requested_freq < policy->min) - dbs_info->requested_freq = policy->min; + if (requested_freq > freq_step) + requested_freq -= freq_step; + else + requested_freq = policy->min; - __cpufreq_driver_target(policy, dbs_info->requested_freq, - CPUFREQ_RELATION_L); - return; + __cpufreq_driver_target(policy, requested_freq, + CPUFREQ_RELATION_LE); + dbs_info->requested_freq = requested_freq; } -} -static void cs_dbs_timer(struct work_struct *work) -{ - struct cs_cpu_dbs_info_s *dbs_info = container_of(work, - struct cs_cpu_dbs_info_s, cdbs.work.work); - unsigned int cpu = dbs_info->cdbs.cur_policy->cpu; - struct cs_cpu_dbs_info_s *core_dbs_info = &per_cpu(cs_cpu_dbs_info, - cpu); - struct dbs_data *dbs_data = dbs_info->cdbs.cur_policy->governor_data; - struct cs_dbs_tuners *cs_tuners = dbs_data->tuners; - int delay = delay_for_sampling_rate(cs_tuners->sampling_rate); - bool modify_all = true; - - mutex_lock(&core_dbs_info->cdbs.timer_mutex); - if (!need_load_eval(&core_dbs_info->cdbs, cs_tuners->sampling_rate)) - modify_all = false; - else - dbs_check_cpu(dbs_data, cpu); - - gov_queue_work(dbs_data, dbs_info->cdbs.cur_policy, delay, modify_all); - mutex_unlock(&core_dbs_info->cdbs.timer_mutex); -} - -static int dbs_cpufreq_notifier(struct notifier_block *nb, unsigned long val, - void *data) -{ - struct cpufreq_freqs *freq = data; - struct cs_cpu_dbs_info_s *dbs_info = - &per_cpu(cs_cpu_dbs_info, freq->cpu); - struct cpufreq_policy *policy; - - if (!dbs_info->enable) - return 0; - - policy = dbs_info->cdbs.cur_policy; - - /* - * we only care if our internally tracked freq moves outside the 'valid' - * ranges of frequency available to us otherwise we do not change it - */ - if (dbs_info->requested_freq > policy->max - || dbs_info->requested_freq < policy->min) - dbs_info->requested_freq = freq->new; - - return 0; + out: + return dbs_data->sampling_rate; } /************************** sysfs interface ************************/ -static struct common_dbs_data cs_dbs_cdata; -static ssize_t store_sampling_down_factor(struct dbs_data *dbs_data, - const char *buf, size_t count) +static ssize_t sampling_down_factor_store(struct gov_attr_set *attr_set, + const char *buf, size_t count) { - struct cs_dbs_tuners *cs_tuners = dbs_data->tuners; + struct dbs_data *dbs_data = to_dbs_data(attr_set); unsigned int input; int ret; - ret = sscanf(buf, "%u", &input); + ret = kstrtouint(buf, 0, &input); - if (ret != 1 || input > MAX_SAMPLING_DOWN_FACTOR || input < 1) + if (ret || input > MAX_SAMPLING_DOWN_FACTOR || input < 1) return -EINVAL; - cs_tuners->sampling_down_factor = input; + dbs_data->sampling_down_factor = input; return count; } -static ssize_t store_sampling_rate(struct dbs_data *dbs_data, const char *buf, - size_t count) +static ssize_t up_threshold_store(struct gov_attr_set *attr_set, + const char *buf, size_t count) { + struct dbs_data *dbs_data = to_dbs_data(attr_set); struct cs_dbs_tuners *cs_tuners = dbs_data->tuners; unsigned int input; int ret; - ret = sscanf(buf, "%u", &input); + ret = kstrtouint(buf, 0, &input); - if (ret != 1) + if (ret || input > 100 || input <= cs_tuners->down_threshold) return -EINVAL; - cs_tuners->sampling_rate = max(input, dbs_data->min_sampling_rate); + dbs_data->up_threshold = input; return count; } -static ssize_t store_up_threshold(struct dbs_data *dbs_data, const char *buf, - size_t count) +static ssize_t down_threshold_store(struct gov_attr_set *attr_set, + const char *buf, size_t count) { + struct dbs_data *dbs_data = to_dbs_data(attr_set); struct cs_dbs_tuners *cs_tuners = dbs_data->tuners; unsigned int input; int ret; - ret = sscanf(buf, "%u", &input); + ret = kstrtouint(buf, 0, &input); - if (ret != 1 || input > 100 || input <= cs_tuners->down_threshold) - return -EINVAL; - - cs_tuners->up_threshold = input; - return count; -} - -static ssize_t store_down_threshold(struct dbs_data *dbs_data, const char *buf, - size_t count) -{ - struct cs_dbs_tuners *cs_tuners = dbs_data->tuners; - unsigned int input; - int ret; - ret = sscanf(buf, "%u", &input); - - /* cannot be lower than 11 otherwise freq will not fall */ - if (ret != 1 || input < 11 || input > 100 || - input >= cs_tuners->up_threshold) + /* cannot be lower than 1 otherwise freq will not fall */ + if (ret || input < 1 || input >= dbs_data->up_threshold) return -EINVAL; cs_tuners->down_threshold = input; return count; } -static ssize_t store_ignore_nice(struct dbs_data *dbs_data, const char *buf, - size_t count) +static ssize_t ignore_nice_load_store(struct gov_attr_set *attr_set, + const char *buf, size_t count) { - struct cs_dbs_tuners *cs_tuners = dbs_data->tuners; - unsigned int input, j; + struct dbs_data *dbs_data = to_dbs_data(attr_set); + unsigned int input; int ret; - ret = sscanf(buf, "%u", &input); - if (ret != 1) - return -EINVAL; + ret = kstrtouint(buf, 0, &input); + if (ret) + return ret; if (input > 1) input = 1; - if (input == cs_tuners->ignore_nice) /* nothing to do */ + if (input == dbs_data->ignore_nice_load) /* nothing to do */ return count; - cs_tuners->ignore_nice = input; + dbs_data->ignore_nice_load = input; /* we need to re-evaluate prev_cpu_idle */ - for_each_online_cpu(j) { - struct cs_cpu_dbs_info_s *dbs_info; - dbs_info = &per_cpu(cs_cpu_dbs_info, j); - dbs_info->cdbs.prev_cpu_idle = get_cpu_idle_time(j, - &dbs_info->cdbs.prev_cpu_wall, 0); - if (cs_tuners->ignore_nice) - dbs_info->cdbs.prev_cpu_nice = - kcpustat_cpu(j).cpustat[CPUTIME_NICE]; - } + gov_update_cpu_data(dbs_data); + return count; } -static ssize_t store_freq_step(struct dbs_data *dbs_data, const char *buf, - size_t count) +static ssize_t freq_step_store(struct gov_attr_set *attr_set, const char *buf, + size_t count) { + struct dbs_data *dbs_data = to_dbs_data(attr_set); struct cs_dbs_tuners *cs_tuners = dbs_data->tuners; unsigned int input; int ret; - ret = sscanf(buf, "%u", &input); + ret = kstrtouint(buf, 0, &input); - if (ret != 1) - return -EINVAL; + if (ret) + return ret; if (input > 100) input = 100; @@ -275,76 +242,61 @@ static ssize_t store_freq_step(struct dbs_data *dbs_data, const char *buf, return count; } -show_store_one(cs, sampling_rate); -show_store_one(cs, sampling_down_factor); -show_store_one(cs, up_threshold); -show_store_one(cs, down_threshold); -show_store_one(cs, ignore_nice); -show_store_one(cs, freq_step); -declare_show_sampling_rate_min(cs); - -gov_sys_pol_attr_rw(sampling_rate); -gov_sys_pol_attr_rw(sampling_down_factor); -gov_sys_pol_attr_rw(up_threshold); -gov_sys_pol_attr_rw(down_threshold); -gov_sys_pol_attr_rw(ignore_nice); -gov_sys_pol_attr_rw(freq_step); -gov_sys_pol_attr_ro(sampling_rate_min); - -static struct attribute *dbs_attributes_gov_sys[] = { - &sampling_rate_min_gov_sys.attr, - &sampling_rate_gov_sys.attr, - &sampling_down_factor_gov_sys.attr, - &up_threshold_gov_sys.attr, - &down_threshold_gov_sys.attr, - &ignore_nice_gov_sys.attr, - &freq_step_gov_sys.attr, +gov_show_one_common(sampling_rate); +gov_show_one_common(sampling_down_factor); +gov_show_one_common(up_threshold); +gov_show_one_common(ignore_nice_load); +gov_show_one(cs, down_threshold); +gov_show_one(cs, freq_step); + +gov_attr_rw(sampling_rate); +gov_attr_rw(sampling_down_factor); +gov_attr_rw(up_threshold); +gov_attr_rw(ignore_nice_load); +gov_attr_rw(down_threshold); +gov_attr_rw(freq_step); + +static struct attribute *cs_attrs[] = { + &sampling_rate.attr, + &sampling_down_factor.attr, + &up_threshold.attr, + &down_threshold.attr, + &ignore_nice_load.attr, + &freq_step.attr, NULL }; +ATTRIBUTE_GROUPS(cs); -static struct attribute_group cs_attr_group_gov_sys = { - .attrs = dbs_attributes_gov_sys, - .name = "conservative", -}; +/************************** sysfs end ************************/ -static struct attribute *dbs_attributes_gov_pol[] = { - &sampling_rate_min_gov_pol.attr, - &sampling_rate_gov_pol.attr, - &sampling_down_factor_gov_pol.attr, - &up_threshold_gov_pol.attr, - &down_threshold_gov_pol.attr, - &ignore_nice_gov_pol.attr, - &freq_step_gov_pol.attr, - NULL -}; +static struct policy_dbs_info *cs_alloc(void) +{ + struct cs_policy_dbs_info *dbs_info; -static struct attribute_group cs_attr_group_gov_pol = { - .attrs = dbs_attributes_gov_pol, - .name = "conservative", -}; + dbs_info = kzalloc(sizeof(*dbs_info), GFP_KERNEL); + return dbs_info ? &dbs_info->policy_dbs : NULL; +} -/************************** sysfs end ************************/ +static void cs_free(struct policy_dbs_info *policy_dbs) +{ + kfree(to_dbs_info(policy_dbs)); +} static int cs_init(struct dbs_data *dbs_data) { struct cs_dbs_tuners *tuners; - tuners = kzalloc(sizeof(struct cs_dbs_tuners), GFP_KERNEL); - if (!tuners) { - pr_err("%s: kzalloc failed\n", __func__); + tuners = kzalloc(sizeof(*tuners), GFP_KERNEL); + if (!tuners) return -ENOMEM; - } - tuners->up_threshold = DEF_FREQUENCY_UP_THRESHOLD; tuners->down_threshold = DEF_FREQUENCY_DOWN_THRESHOLD; - tuners->sampling_down_factor = DEF_SAMPLING_DOWN_FACTOR; - tuners->ignore_nice = 0; tuners->freq_step = DEF_FREQUENCY_STEP; - + dbs_data->up_threshold = DEF_FREQUENCY_UP_THRESHOLD; + dbs_data->sampling_down_factor = DEF_SAMPLING_DOWN_FACTOR; + dbs_data->ignore_nice_load = 0; dbs_data->tuners = tuners; - dbs_data->min_sampling_rate = MIN_SAMPLING_RATE_RATIO * - jiffies_to_usecs(10); - mutex_init(&dbs_data->mutex); + return 0; } @@ -353,54 +305,26 @@ static void cs_exit(struct dbs_data *dbs_data) kfree(dbs_data->tuners); } -define_get_cpu_dbs_routines(cs_cpu_dbs_info); - -static struct notifier_block cs_cpufreq_notifier_block = { - .notifier_call = dbs_cpufreq_notifier, -}; +static void cs_start(struct cpufreq_policy *policy) +{ + struct cs_policy_dbs_info *dbs_info = to_dbs_info(policy->governor_data); -static struct cs_ops cs_ops = { - .notifier_block = &cs_cpufreq_notifier_block, -}; + dbs_info->down_skip = 0; + dbs_info->requested_freq = policy->cur; +} -static struct common_dbs_data cs_dbs_cdata = { - .governor = GOV_CONSERVATIVE, - .attr_group_gov_sys = &cs_attr_group_gov_sys, - .attr_group_gov_pol = &cs_attr_group_gov_pol, - .get_cpu_cdbs = get_cpu_cdbs, - .get_cpu_dbs_info_s = get_cpu_dbs_info_s, - .gov_dbs_timer = cs_dbs_timer, - .gov_check_cpu = cs_check_cpu, - .gov_ops = &cs_ops, +static struct dbs_governor cs_governor = { + .gov = CPUFREQ_DBS_GOVERNOR_INITIALIZER("conservative"), + .kobj_type = { .default_groups = cs_groups }, + .gov_dbs_update = cs_dbs_update, + .alloc = cs_alloc, + .free = cs_free, .init = cs_init, .exit = cs_exit, + .start = cs_start, }; -static int cs_cpufreq_governor_dbs(struct cpufreq_policy *policy, - unsigned int event) -{ - return cpufreq_governor_dbs(policy, &cs_dbs_cdata, event); -} - -#ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_CONSERVATIVE -static -#endif -struct cpufreq_governor cpufreq_gov_conservative = { - .name = "conservative", - .governor = cs_cpufreq_governor_dbs, - .max_transition_latency = TRANSITION_LATENCY_LIMIT, - .owner = THIS_MODULE, -}; - -static int __init cpufreq_gov_dbs_init(void) -{ - return cpufreq_register_governor(&cpufreq_gov_conservative); -} - -static void __exit cpufreq_gov_dbs_exit(void) -{ - cpufreq_unregister_governor(&cpufreq_gov_conservative); -} +#define CPU_FREQ_GOV_CONSERVATIVE (cs_governor.gov) MODULE_AUTHOR("Alexander Clouter <alex@digriz.org.uk>"); MODULE_DESCRIPTION("'cpufreq_conservative' - A dynamic cpufreq governor for " @@ -409,8 +333,11 @@ MODULE_DESCRIPTION("'cpufreq_conservative' - A dynamic cpufreq governor for " MODULE_LICENSE("GPL"); #ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_CONSERVATIVE -fs_initcall(cpufreq_gov_dbs_init); -#else -module_init(cpufreq_gov_dbs_init); +struct cpufreq_governor *cpufreq_default_governor(void) +{ + return &CPU_FREQ_GOV_CONSERVATIVE; +} #endif -module_exit(cpufreq_gov_dbs_exit); + +cpufreq_governor_init(CPU_FREQ_GOV_CONSERVATIVE); +cpufreq_governor_exit(CPU_FREQ_GOV_CONSERVATIVE); diff --git a/drivers/cpufreq/cpufreq_governor.c b/drivers/cpufreq/cpufreq_governor.c index 464587697561..1a7fcaf39cc9 100644 --- a/drivers/cpufreq/cpufreq_governor.c +++ b/drivers/cpufreq/cpufreq_governor.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * drivers/cpufreq/cpufreq_governor.c * @@ -8,382 +9,574 @@ * (C) 2003 Jun Nakajima <jun.nakajima@intel.com> * (C) 2009 Alexander Clouter <alex@digriz.org.uk> * (c) 2012 Viresh Kumar <viresh.kumar@linaro.org> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -#include <asm/cputime.h> -#include <linux/cpufreq.h> -#include <linux/cpumask.h> #include <linux/export.h> #include <linux/kernel_stat.h> -#include <linux/mutex.h> #include <linux/slab.h> -#include <linux/types.h> -#include <linux/workqueue.h> -#include <linux/cpu.h> #include "cpufreq_governor.h" -static struct attribute_group *get_sysfs_attr(struct dbs_data *dbs_data) +#define CPUFREQ_DBS_MIN_SAMPLING_INTERVAL (2 * TICK_NSEC / NSEC_PER_USEC) + +static DEFINE_PER_CPU(struct cpu_dbs_info, cpu_dbs); + +static DEFINE_MUTEX(gov_dbs_data_mutex); + +/* Common sysfs tunables */ +/* + * sampling_rate_store - update sampling rate effective immediately if needed. + * + * If new rate is smaller than the old, simply updating + * dbs.sampling_rate might not be appropriate. For example, if the + * original sampling_rate was 1 second and the requested new sampling rate is 10 + * ms because the user needs immediate reaction from ondemand governor, but not + * sure if higher frequency will be required or not, then, the governor may + * change the sampling rate too late; up to 1 second later. Thus, if we are + * reducing the sampling rate, we need to make the new value effective + * immediately. + * + * This must be called with dbs_data->mutex held, otherwise traversing + * policy_dbs_list isn't safe. + */ +ssize_t sampling_rate_store(struct gov_attr_set *attr_set, const char *buf, + size_t count) { - if (have_governor_per_policy()) - return dbs_data->cdata->attr_group_gov_pol; - else - return dbs_data->cdata->attr_group_gov_sys; + struct dbs_data *dbs_data = to_dbs_data(attr_set); + struct policy_dbs_info *policy_dbs; + unsigned int sampling_interval; + int ret; + + ret = sscanf(buf, "%u", &sampling_interval); + if (ret != 1 || sampling_interval < CPUFREQ_DBS_MIN_SAMPLING_INTERVAL) + return -EINVAL; + + dbs_data->sampling_rate = sampling_interval; + + /* + * We are operating under dbs_data->mutex and so the list and its + * entries can't be freed concurrently. + */ + list_for_each_entry(policy_dbs, &attr_set->policy_list, list) { + mutex_lock(&policy_dbs->update_mutex); + /* + * On 32-bit architectures this may race with the + * sample_delay_ns read in dbs_update_util_handler(), but that + * really doesn't matter. If the read returns a value that's + * too big, the sample will be skipped, but the next invocation + * of dbs_update_util_handler() (when the update has been + * completed) will take a sample. + * + * If this runs in parallel with dbs_work_handler(), we may end + * up overwriting the sample_delay_ns value that it has just + * written, but it will be corrected next time a sample is + * taken, so it shouldn't be significant. + */ + gov_update_sample_delay(policy_dbs, 0); + mutex_unlock(&policy_dbs->update_mutex); + } + + return count; } +EXPORT_SYMBOL_GPL(sampling_rate_store); -void dbs_check_cpu(struct dbs_data *dbs_data, int cpu) +/** + * gov_update_cpu_data - Update CPU load data. + * @dbs_data: Top-level governor data pointer. + * + * Update CPU load data for all CPUs in the domain governed by @dbs_data + * (that may be a single policy or a bunch of them if governor tunables are + * system-wide). + * + * Call under the @dbs_data mutex. + */ +void gov_update_cpu_data(struct dbs_data *dbs_data) { - struct cpu_dbs_common_info *cdbs = dbs_data->cdata->get_cpu_cdbs(cpu); - struct od_dbs_tuners *od_tuners = dbs_data->tuners; - struct cs_dbs_tuners *cs_tuners = dbs_data->tuners; - struct cpufreq_policy *policy; - unsigned int max_load = 0; - unsigned int ignore_nice; - unsigned int j; + struct policy_dbs_info *policy_dbs; + + list_for_each_entry(policy_dbs, &dbs_data->attr_set.policy_list, list) { + unsigned int j; - if (dbs_data->cdata->governor == GOV_ONDEMAND) - ignore_nice = od_tuners->ignore_nice; - else - ignore_nice = cs_tuners->ignore_nice; + for_each_cpu(j, policy_dbs->policy->cpus) { + struct cpu_dbs_info *j_cdbs = &per_cpu(cpu_dbs, j); - policy = cdbs->cur_policy; + j_cdbs->prev_cpu_idle = get_cpu_idle_time(j, &j_cdbs->prev_update_time, + dbs_data->io_is_busy); + if (dbs_data->ignore_nice_load) + j_cdbs->prev_cpu_nice = kcpustat_field(&kcpustat_cpu(j), CPUTIME_NICE, j); + } + } +} +EXPORT_SYMBOL_GPL(gov_update_cpu_data); - /* Get Absolute Load (in terms of freq for ondemand gov) */ +unsigned int dbs_update(struct cpufreq_policy *policy) +{ + struct policy_dbs_info *policy_dbs = policy->governor_data; + struct dbs_data *dbs_data = policy_dbs->dbs_data; + unsigned int ignore_nice = dbs_data->ignore_nice_load; + unsigned int max_load = 0, idle_periods = UINT_MAX; + unsigned int sampling_rate, io_busy, j; + + /* + * Sometimes governors may use an additional multiplier to increase + * sample delays temporarily. Apply that multiplier to sampling_rate + * so as to keep the wake-up-from-idle detection logic a bit + * conservative. + */ + sampling_rate = dbs_data->sampling_rate * policy_dbs->rate_mult; + /* + * For the purpose of ondemand, waiting for disk IO is an indication + * that you're performance critical, and not that the system is actually + * idle, so do not add the iowait time to the CPU idle time then. + */ + io_busy = dbs_data->io_is_busy; + + /* Get Absolute Load */ for_each_cpu(j, policy->cpus) { - struct cpu_dbs_common_info *j_cdbs; - u64 cur_wall_time, cur_idle_time; - unsigned int idle_time, wall_time; + struct cpu_dbs_info *j_cdbs = &per_cpu(cpu_dbs, j); + u64 update_time, cur_idle_time; + unsigned int idle_time, time_elapsed; unsigned int load; - int io_busy = 0; - j_cdbs = dbs_data->cdata->get_cpu_cdbs(j); + cur_idle_time = get_cpu_idle_time(j, &update_time, io_busy); + + time_elapsed = update_time - j_cdbs->prev_update_time; + j_cdbs->prev_update_time = update_time; /* - * For the purpose of ondemand, waiting for disk IO is - * an indication that you're performance critical, and - * not that the system is actually idle. So do not add - * the iowait time to the cpu idle time. + * cur_idle_time could be smaller than j_cdbs->prev_cpu_idle if + * it's obtained from get_cpu_idle_time_jiffy() when NOHZ is + * off, where idle_time is calculated by the difference between + * time elapsed in jiffies and "busy time" obtained from CPU + * statistics. If a CPU is 100% busy, the time elapsed and busy + * time should grow with the same amount in two consecutive + * samples, but in practice there could be a tiny difference, + * making the accumulated idle time decrease sometimes. Hence, + * in this case, idle_time should be regarded as 0 in order to + * make the further process correct. */ - if (dbs_data->cdata->governor == GOV_ONDEMAND) - io_busy = od_tuners->io_is_busy; - cur_idle_time = get_cpu_idle_time(j, &cur_wall_time, io_busy); - - wall_time = (unsigned int) - (cur_wall_time - j_cdbs->prev_cpu_wall); - j_cdbs->prev_cpu_wall = cur_wall_time; + if (cur_idle_time > j_cdbs->prev_cpu_idle) + idle_time = cur_idle_time - j_cdbs->prev_cpu_idle; + else + idle_time = 0; - idle_time = (unsigned int) - (cur_idle_time - j_cdbs->prev_cpu_idle); j_cdbs->prev_cpu_idle = cur_idle_time; if (ignore_nice) { - u64 cur_nice; - unsigned long cur_nice_jiffies; + u64 cur_nice = kcpustat_field(&kcpustat_cpu(j), CPUTIME_NICE, j); - cur_nice = kcpustat_cpu(j).cpustat[CPUTIME_NICE] - - cdbs->prev_cpu_nice; + idle_time += div_u64(cur_nice - j_cdbs->prev_cpu_nice, NSEC_PER_USEC); + j_cdbs->prev_cpu_nice = cur_nice; + } + + if (unlikely(!time_elapsed)) { + /* + * That can only happen when this function is called + * twice in a row with a very short interval between the + * calls, so the previous load value can be used then. + */ + load = j_cdbs->prev_load; + } else if (unlikely(idle_time > 2 * sampling_rate && + j_cdbs->prev_load)) { /* - * Assumption: nice time between sampling periods will - * be less than 2^32 jiffies for 32 bit sys + * If the CPU had gone completely idle and a task has + * just woken up on this CPU now, it would be unfair to + * calculate 'load' the usual way for this elapsed + * time-window, because it would show near-zero load, + * irrespective of how CPU intensive that task actually + * was. This is undesirable for latency-sensitive bursty + * workloads. + * + * To avoid this, reuse the 'load' from the previous + * time-window and give this task a chance to start with + * a reasonably high CPU frequency. However, that + * shouldn't be over-done, lest we get stuck at a high + * load (high frequency) for too long, even when the + * current system load has actually dropped down, so + * clear prev_load to guarantee that the load will be + * computed again next time. + * + * Detecting this situation is easy: an unusually large + * 'idle_time' (as compared to the sampling rate) + * indicates this scenario. */ - cur_nice_jiffies = (unsigned long) - cputime64_to_jiffies64(cur_nice); + load = j_cdbs->prev_load; + j_cdbs->prev_load = 0; + } else { + if (time_elapsed > idle_time) + load = 100 * (time_elapsed - idle_time) / time_elapsed; + else + load = 0; - cdbs->prev_cpu_nice = - kcpustat_cpu(j).cpustat[CPUTIME_NICE]; - idle_time += jiffies_to_usecs(cur_nice_jiffies); + j_cdbs->prev_load = load; } - if (unlikely(!wall_time || wall_time < idle_time)) - continue; - - load = 100 * (wall_time - idle_time) / wall_time; + if (unlikely(idle_time > 2 * sampling_rate)) { + unsigned int periods = idle_time / sampling_rate; - if (dbs_data->cdata->governor == GOV_ONDEMAND) { - int freq_avg = __cpufreq_driver_getavg(policy, j); - if (freq_avg <= 0) - freq_avg = policy->cur; - - load *= freq_avg; + if (periods < idle_periods) + idle_periods = periods; } if (load > max_load) max_load = load; } - dbs_data->cdata->gov_check_cpu(cpu, max_load); + policy_dbs->idle_periods = idle_periods; + + return max_load; +} +EXPORT_SYMBOL_GPL(dbs_update); + +static void dbs_work_handler(struct work_struct *work) +{ + struct policy_dbs_info *policy_dbs; + struct cpufreq_policy *policy; + struct dbs_governor *gov; + + policy_dbs = container_of(work, struct policy_dbs_info, work); + policy = policy_dbs->policy; + gov = dbs_governor_of(policy); + + /* + * Make sure cpufreq_governor_limits() isn't evaluating load or the + * ondemand governor isn't updating the sampling rate in parallel. + */ + mutex_lock(&policy_dbs->update_mutex); + gov_update_sample_delay(policy_dbs, gov->gov_dbs_update(policy)); + mutex_unlock(&policy_dbs->update_mutex); + + /* Allow the utilization update handler to queue up more work. */ + atomic_set(&policy_dbs->work_count, 0); + /* + * If the update below is reordered with respect to the sample delay + * modification, the utilization update handler may end up using a stale + * sample delay value. + */ + smp_wmb(); + policy_dbs->work_in_progress = false; } -EXPORT_SYMBOL_GPL(dbs_check_cpu); -static inline void __gov_queue_work(int cpu, struct dbs_data *dbs_data, - unsigned int delay) +static void dbs_irq_work(struct irq_work *irq_work) { - struct cpu_dbs_common_info *cdbs = dbs_data->cdata->get_cpu_cdbs(cpu); + struct policy_dbs_info *policy_dbs; - mod_delayed_work_on(cpu, system_wq, &cdbs->work, delay); + policy_dbs = container_of(irq_work, struct policy_dbs_info, irq_work); + schedule_work_on(smp_processor_id(), &policy_dbs->work); } -void gov_queue_work(struct dbs_data *dbs_data, struct cpufreq_policy *policy, - unsigned int delay, bool all_cpus) +static void dbs_update_util_handler(struct update_util_data *data, u64 time, + unsigned int flags) { - int i; + struct cpu_dbs_info *cdbs = container_of(data, struct cpu_dbs_info, update_util); + struct policy_dbs_info *policy_dbs = cdbs->policy_dbs; + u64 delta_ns, lst; + + if (!cpufreq_this_cpu_can_update(policy_dbs->policy)) + return; + + /* + * The work may not be allowed to be queued up right now. + * Possible reasons: + * - Work has already been queued up or is in progress. + * - It is too early (too little time from the previous sample). + */ + if (policy_dbs->work_in_progress) + return; + + /* + * If the reads below are reordered before the check above, the value + * of sample_delay_ns used in the computation may be stale. + */ + smp_rmb(); + lst = READ_ONCE(policy_dbs->last_sample_time); + delta_ns = time - lst; + if ((s64)delta_ns < policy_dbs->sample_delay_ns) + return; + + /* + * If the policy is not shared, the irq_work may be queued up right away + * at this point. Otherwise, we need to ensure that only one of the + * CPUs sharing the policy will do that. + */ + if (policy_dbs->is_shared) { + if (!atomic_add_unless(&policy_dbs->work_count, 1, 1)) + return; - if (!all_cpus) { - __gov_queue_work(smp_processor_id(), dbs_data, delay); - } else { - get_online_cpus(); - for_each_cpu(i, policy->cpus) - __gov_queue_work(i, dbs_data, delay); - put_online_cpus(); + /* + * If another CPU updated last_sample_time in the meantime, we + * shouldn't be here, so clear the work counter and bail out. + */ + if (unlikely(lst != READ_ONCE(policy_dbs->last_sample_time))) { + atomic_set(&policy_dbs->work_count, 0); + return; + } } + + policy_dbs->last_sample_time = time; + policy_dbs->work_in_progress = true; + irq_work_queue(&policy_dbs->irq_work); } -EXPORT_SYMBOL_GPL(gov_queue_work); -static inline void gov_cancel_work(struct dbs_data *dbs_data, - struct cpufreq_policy *policy) +static void gov_set_update_util(struct policy_dbs_info *policy_dbs, + unsigned int delay_us) { - struct cpu_dbs_common_info *cdbs; - int i; + struct cpufreq_policy *policy = policy_dbs->policy; + int cpu; - for_each_cpu(i, policy->cpus) { - cdbs = dbs_data->cdata->get_cpu_cdbs(i); - cancel_delayed_work_sync(&cdbs->work); + gov_update_sample_delay(policy_dbs, delay_us); + policy_dbs->last_sample_time = 0; + + for_each_cpu(cpu, policy->cpus) { + struct cpu_dbs_info *cdbs = &per_cpu(cpu_dbs, cpu); + + cpufreq_add_update_util_hook(cpu, &cdbs->update_util, + dbs_update_util_handler); } } -/* Will return if we need to evaluate cpu load again or not */ -bool need_load_eval(struct cpu_dbs_common_info *cdbs, - unsigned int sampling_rate) +static inline void gov_clear_update_util(struct cpufreq_policy *policy) { - if (policy_is_shared(cdbs->cur_policy)) { - ktime_t time_now = ktime_get(); - s64 delta_us = ktime_us_delta(time_now, cdbs->time_stamp); + int i; - /* Do nothing if we recently have sampled */ - if (delta_us < (s64)(sampling_rate / 2)) - return false; - else - cdbs->time_stamp = time_now; - } + for_each_cpu(i, policy->cpus) + cpufreq_remove_update_util_hook(i); + + synchronize_rcu(); +} + +static struct policy_dbs_info *alloc_policy_dbs_info(struct cpufreq_policy *policy, + struct dbs_governor *gov) +{ + struct policy_dbs_info *policy_dbs; + int j; + + /* Allocate memory for per-policy governor data. */ + policy_dbs = gov->alloc(); + if (!policy_dbs) + return NULL; + + policy_dbs->policy = policy; + mutex_init(&policy_dbs->update_mutex); + atomic_set(&policy_dbs->work_count, 0); + init_irq_work(&policy_dbs->irq_work, dbs_irq_work); + INIT_WORK(&policy_dbs->work, dbs_work_handler); + + /* Set policy_dbs for all CPUs, online+offline */ + for_each_cpu(j, policy->related_cpus) { + struct cpu_dbs_info *j_cdbs = &per_cpu(cpu_dbs, j); - return true; + j_cdbs->policy_dbs = policy_dbs; + } + return policy_dbs; } -EXPORT_SYMBOL_GPL(need_load_eval); -static void set_sampling_rate(struct dbs_data *dbs_data, - unsigned int sampling_rate) +static void free_policy_dbs_info(struct policy_dbs_info *policy_dbs, + struct dbs_governor *gov) { - if (dbs_data->cdata->governor == GOV_CONSERVATIVE) { - struct cs_dbs_tuners *cs_tuners = dbs_data->tuners; - cs_tuners->sampling_rate = sampling_rate; - } else { - struct od_dbs_tuners *od_tuners = dbs_data->tuners; - od_tuners->sampling_rate = sampling_rate; + int j; + + mutex_destroy(&policy_dbs->update_mutex); + + for_each_cpu(j, policy_dbs->policy->related_cpus) { + struct cpu_dbs_info *j_cdbs = &per_cpu(cpu_dbs, j); + + j_cdbs->policy_dbs = NULL; + j_cdbs->update_util.func = NULL; } + gov->free(policy_dbs); +} + +static void cpufreq_dbs_data_release(struct kobject *kobj) +{ + struct dbs_data *dbs_data = to_dbs_data(to_gov_attr_set(kobj)); + struct dbs_governor *gov = dbs_data->gov; + + gov->exit(dbs_data); + kfree(dbs_data); } -int cpufreq_governor_dbs(struct cpufreq_policy *policy, - struct common_dbs_data *cdata, unsigned int event) +int cpufreq_dbs_governor_init(struct cpufreq_policy *policy) { + struct dbs_governor *gov = dbs_governor_of(policy); struct dbs_data *dbs_data; - struct od_cpu_dbs_info_s *od_dbs_info = NULL; - struct cs_cpu_dbs_info_s *cs_dbs_info = NULL; - struct od_ops *od_ops = NULL; - struct od_dbs_tuners *od_tuners = NULL; - struct cs_dbs_tuners *cs_tuners = NULL; - struct cpu_dbs_common_info *cpu_cdbs; - unsigned int sampling_rate, latency, ignore_nice, j, cpu = policy->cpu; - int io_busy = 0; - int rc; - - if (have_governor_per_policy()) - dbs_data = policy->governor_data; - else - dbs_data = cdata->gdbs_data; - - WARN_ON(!dbs_data && (event != CPUFREQ_GOV_POLICY_INIT)); - - switch (event) { - case CPUFREQ_GOV_POLICY_INIT: - if (have_governor_per_policy()) { - WARN_ON(dbs_data); - } else if (dbs_data) { - dbs_data->usage_count++; - policy->governor_data = dbs_data; - return 0; - } + struct policy_dbs_info *policy_dbs; + int ret = 0; - dbs_data = kzalloc(sizeof(*dbs_data), GFP_KERNEL); - if (!dbs_data) { - pr_err("%s: POLICY_INIT: kzalloc failed\n", __func__); - return -ENOMEM; - } + /* State should be equivalent to EXIT */ + if (policy->governor_data) + return -EBUSY; - dbs_data->cdata = cdata; - dbs_data->usage_count = 1; - rc = cdata->init(dbs_data); - if (rc) { - pr_err("%s: POLICY_INIT: init() failed\n", __func__); - kfree(dbs_data); - return rc; - } + policy_dbs = alloc_policy_dbs_info(policy, gov); + if (!policy_dbs) + return -ENOMEM; - if (!have_governor_per_policy()) - WARN_ON(cpufreq_get_global_kobject()); + /* Protect gov->gdbs_data against concurrent updates. */ + mutex_lock(&gov_dbs_data_mutex); - rc = sysfs_create_group(get_governor_parent_kobj(policy), - get_sysfs_attr(dbs_data)); - if (rc) { - cdata->exit(dbs_data); - kfree(dbs_data); - return rc; + dbs_data = gov->gdbs_data; + if (dbs_data) { + if (WARN_ON(have_governor_per_policy())) { + ret = -EINVAL; + goto free_policy_dbs_info; } + policy_dbs->dbs_data = dbs_data; + policy->governor_data = policy_dbs; + + gov_attr_set_get(&dbs_data->attr_set, &policy_dbs->list); + goto out; + } - policy->governor_data = dbs_data; + dbs_data = kzalloc(sizeof(*dbs_data), GFP_KERNEL); + if (!dbs_data) { + ret = -ENOMEM; + goto free_policy_dbs_info; + } - /* policy latency is in nS. Convert it to uS first */ - latency = policy->cpuinfo.transition_latency / 1000; - if (latency == 0) - latency = 1; + dbs_data->gov = gov; + gov_attr_set_init(&dbs_data->attr_set, &policy_dbs->list); - /* Bring kernel and HW constraints together */ - dbs_data->min_sampling_rate = max(dbs_data->min_sampling_rate, - MIN_LATENCY_MULTIPLIER * latency); - set_sampling_rate(dbs_data, max(dbs_data->min_sampling_rate, - latency * LATENCY_MULTIPLIER)); + ret = gov->init(dbs_data); + if (ret) + goto free_dbs_data; - if ((cdata->governor == GOV_CONSERVATIVE) && - (!policy->governor->initialized)) { - struct cs_ops *cs_ops = dbs_data->cdata->gov_ops; + /* + * The sampling interval should not be less than the transition latency + * of the CPU and it also cannot be too small for dbs_update() to work + * correctly. + */ + dbs_data->sampling_rate = max_t(unsigned int, + CPUFREQ_DBS_MIN_SAMPLING_INTERVAL, + cpufreq_policy_transition_delay_us(policy)); - cpufreq_register_notifier(cs_ops->notifier_block, - CPUFREQ_TRANSITION_NOTIFIER); - } + if (!have_governor_per_policy()) + gov->gdbs_data = dbs_data; - if (!have_governor_per_policy()) - cdata->gdbs_data = dbs_data; + policy_dbs->dbs_data = dbs_data; + policy->governor_data = policy_dbs; - return 0; - case CPUFREQ_GOV_POLICY_EXIT: - if (!--dbs_data->usage_count) { - sysfs_remove_group(get_governor_parent_kobj(policy), - get_sysfs_attr(dbs_data)); + gov->kobj_type.sysfs_ops = &governor_sysfs_ops; + gov->kobj_type.release = cpufreq_dbs_data_release; + ret = kobject_init_and_add(&dbs_data->attr_set.kobj, &gov->kobj_type, + get_governor_parent_kobj(policy), + "%s", gov->gov.name); + if (!ret) + goto out; - if (!have_governor_per_policy()) - cpufreq_put_global_kobject(); + /* Failure, so roll back. */ + pr_err("initialization failed (dbs_data kobject init error %d)\n", ret); - if ((dbs_data->cdata->governor == GOV_CONSERVATIVE) && - (policy->governor->initialized == 1)) { - struct cs_ops *cs_ops = dbs_data->cdata->gov_ops; + kobject_put(&dbs_data->attr_set.kobj); - cpufreq_unregister_notifier(cs_ops->notifier_block, - CPUFREQ_TRANSITION_NOTIFIER); - } + policy->governor_data = NULL; - cdata->exit(dbs_data); - kfree(dbs_data); - cdata->gdbs_data = NULL; - } + if (!have_governor_per_policy()) + gov->gdbs_data = NULL; + gov->exit(dbs_data); - policy->governor_data = NULL; - return 0; - } +free_dbs_data: + kfree(dbs_data); - cpu_cdbs = dbs_data->cdata->get_cpu_cdbs(cpu); - - if (dbs_data->cdata->governor == GOV_CONSERVATIVE) { - cs_tuners = dbs_data->tuners; - cs_dbs_info = dbs_data->cdata->get_cpu_dbs_info_s(cpu); - sampling_rate = cs_tuners->sampling_rate; - ignore_nice = cs_tuners->ignore_nice; - } else { - od_tuners = dbs_data->tuners; - od_dbs_info = dbs_data->cdata->get_cpu_dbs_info_s(cpu); - sampling_rate = od_tuners->sampling_rate; - ignore_nice = od_tuners->ignore_nice; - od_ops = dbs_data->cdata->gov_ops; - io_busy = od_tuners->io_is_busy; - } +free_policy_dbs_info: + free_policy_dbs_info(policy_dbs, gov); - switch (event) { - case CPUFREQ_GOV_START: - if (!policy->cur) - return -EINVAL; +out: + mutex_unlock(&gov_dbs_data_mutex); + return ret; +} +EXPORT_SYMBOL_GPL(cpufreq_dbs_governor_init); - mutex_lock(&dbs_data->mutex); +void cpufreq_dbs_governor_exit(struct cpufreq_policy *policy) +{ + struct dbs_governor *gov = dbs_governor_of(policy); + struct policy_dbs_info *policy_dbs = policy->governor_data; + struct dbs_data *dbs_data = policy_dbs->dbs_data; + unsigned int count; - for_each_cpu(j, policy->cpus) { - struct cpu_dbs_common_info *j_cdbs = - dbs_data->cdata->get_cpu_cdbs(j); + /* Protect gov->gdbs_data against concurrent updates. */ + mutex_lock(&gov_dbs_data_mutex); - j_cdbs->cpu = j; - j_cdbs->cur_policy = policy; - j_cdbs->prev_cpu_idle = get_cpu_idle_time(j, - &j_cdbs->prev_cpu_wall, io_busy); - if (ignore_nice) - j_cdbs->prev_cpu_nice = - kcpustat_cpu(j).cpustat[CPUTIME_NICE]; + count = gov_attr_set_put(&dbs_data->attr_set, &policy_dbs->list); - mutex_init(&j_cdbs->timer_mutex); - INIT_DEFERRABLE_WORK(&j_cdbs->work, - dbs_data->cdata->gov_dbs_timer); - } + policy->governor_data = NULL; - /* - * conservative does not implement micro like ondemand - * governor, thus we are bound to jiffes/HZ - */ - if (dbs_data->cdata->governor == GOV_CONSERVATIVE) { - cs_dbs_info->down_skip = 0; - cs_dbs_info->enable = 1; - cs_dbs_info->requested_freq = policy->cur; - } else { - od_dbs_info->rate_mult = 1; - od_dbs_info->sample_type = OD_NORMAL_SAMPLE; - od_ops->powersave_bias_init_cpu(cpu); - } + if (!count && !have_governor_per_policy()) + gov->gdbs_data = NULL; - mutex_unlock(&dbs_data->mutex); + free_policy_dbs_info(policy_dbs, gov); - /* Initiate timer time stamp */ - cpu_cdbs->time_stamp = ktime_get(); + mutex_unlock(&gov_dbs_data_mutex); +} +EXPORT_SYMBOL_GPL(cpufreq_dbs_governor_exit); - gov_queue_work(dbs_data, policy, - delay_for_sampling_rate(sampling_rate), true); - break; +int cpufreq_dbs_governor_start(struct cpufreq_policy *policy) +{ + struct dbs_governor *gov = dbs_governor_of(policy); + struct policy_dbs_info *policy_dbs = policy->governor_data; + struct dbs_data *dbs_data = policy_dbs->dbs_data; + unsigned int sampling_rate, ignore_nice, j; + unsigned int io_busy; - case CPUFREQ_GOV_STOP: - if (dbs_data->cdata->governor == GOV_CONSERVATIVE) - cs_dbs_info->enable = 0; + if (!policy->cur) + return -EINVAL; - gov_cancel_work(dbs_data, policy); + policy_dbs->is_shared = policy_is_shared(policy); + policy_dbs->rate_mult = 1; - mutex_lock(&dbs_data->mutex); - mutex_destroy(&cpu_cdbs->timer_mutex); - cpu_cdbs->cur_policy = NULL; + sampling_rate = dbs_data->sampling_rate; + ignore_nice = dbs_data->ignore_nice_load; + io_busy = dbs_data->io_is_busy; - mutex_unlock(&dbs_data->mutex); + for_each_cpu(j, policy->cpus) { + struct cpu_dbs_info *j_cdbs = &per_cpu(cpu_dbs, j); - break; + j_cdbs->prev_cpu_idle = get_cpu_idle_time(j, &j_cdbs->prev_update_time, io_busy); + /* + * Make the first invocation of dbs_update() compute the load. + */ + j_cdbs->prev_load = 0; - case CPUFREQ_GOV_LIMITS: - mutex_lock(&cpu_cdbs->timer_mutex); - if (policy->max < cpu_cdbs->cur_policy->cur) - __cpufreq_driver_target(cpu_cdbs->cur_policy, - policy->max, CPUFREQ_RELATION_H); - else if (policy->min > cpu_cdbs->cur_policy->cur) - __cpufreq_driver_target(cpu_cdbs->cur_policy, - policy->min, CPUFREQ_RELATION_L); - dbs_check_cpu(dbs_data, cpu); - mutex_unlock(&cpu_cdbs->timer_mutex); - break; + if (ignore_nice) + j_cdbs->prev_cpu_nice = kcpustat_field(&kcpustat_cpu(j), CPUTIME_NICE, j); } + + gov->start(policy); + + gov_set_update_util(policy_dbs, sampling_rate); return 0; } -EXPORT_SYMBOL_GPL(cpufreq_governor_dbs); +EXPORT_SYMBOL_GPL(cpufreq_dbs_governor_start); + +void cpufreq_dbs_governor_stop(struct cpufreq_policy *policy) +{ + struct policy_dbs_info *policy_dbs = policy->governor_data; + + gov_clear_update_util(policy_dbs->policy); + irq_work_sync(&policy_dbs->irq_work); + cancel_work_sync(&policy_dbs->work); + atomic_set(&policy_dbs->work_count, 0); + policy_dbs->work_in_progress = false; +} +EXPORT_SYMBOL_GPL(cpufreq_dbs_governor_stop); + +void cpufreq_dbs_governor_limits(struct cpufreq_policy *policy) +{ + struct policy_dbs_info *policy_dbs; + + /* Protect gov->gdbs_data against cpufreq_dbs_governor_exit() */ + mutex_lock(&gov_dbs_data_mutex); + policy_dbs = policy->governor_data; + if (!policy_dbs) + goto out; + + mutex_lock(&policy_dbs->update_mutex); + cpufreq_policy_apply_limits(policy); + gov_update_sample_delay(policy_dbs, 0); + mutex_unlock(&policy_dbs->update_mutex); + +out: + mutex_unlock(&gov_dbs_data_mutex); +} +EXPORT_SYMBOL_GPL(cpufreq_dbs_governor_limits); diff --git a/drivers/cpufreq/cpufreq_governor.h b/drivers/cpufreq/cpufreq_governor.h index 6663ec3b3056..168c23fd7fca 100644 --- a/drivers/cpufreq/cpufreq_governor.h +++ b/drivers/cpufreq/cpufreq_governor.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * drivers/cpufreq/cpufreq_governor.h * @@ -8,263 +9,173 @@ * (C) 2003 Jun Nakajima <jun.nakajima@intel.com> * (C) 2009 Alexander Clouter <alex@digriz.org.uk> * (c) 2012 Viresh Kumar <viresh.kumar@linaro.org> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #ifndef _CPUFREQ_GOVERNOR_H #define _CPUFREQ_GOVERNOR_H +#include <linux/atomic.h> +#include <linux/irq_work.h> #include <linux/cpufreq.h> -#include <linux/kobject.h> +#include <linux/sched/cpufreq.h> +#include <linux/kernel_stat.h> +#include <linux/module.h> #include <linux/mutex.h> -#include <linux/workqueue.h> -#include <linux/sysfs.h> - -/* - * The polling frequency depends on the capability of the processor. Default - * polling frequency is 1000 times the transition latency of the processor. The - * governor will work on any processor with transition latency <= 10mS, using - * appropriate sampling rate. - * - * For CPUs with transition latency > 10mS (mostly drivers with CPUFREQ_ETERNAL) - * this governor will not work. All times here are in uS. - */ -#define MIN_SAMPLING_RATE_RATIO (2) -#define LATENCY_MULTIPLIER (1000) -#define MIN_LATENCY_MULTIPLIER (20) -#define TRANSITION_LATENCY_LIMIT (10 * 1000 * 1000) /* Ondemand Sampling types */ enum {OD_NORMAL_SAMPLE, OD_SUB_SAMPLE}; /* - * Macro for creating governors sysfs routines - * - * - gov_sys: One governor instance per whole system - * - gov_pol: One governor instance per policy + * Abbreviations: + * dbs: used as a shortform for demand based switching It helps to keep variable + * names smaller, simpler + * cdbs: common dbs + * od_*: On-demand governor + * cs_*: Conservative governor */ -/* Create attributes */ -#define gov_sys_attr_ro(_name) \ -static struct global_attr _name##_gov_sys = \ -__ATTR(_name, 0444, show_##_name##_gov_sys, NULL) - -#define gov_sys_attr_rw(_name) \ -static struct global_attr _name##_gov_sys = \ -__ATTR(_name, 0644, show_##_name##_gov_sys, store_##_name##_gov_sys) - -#define gov_pol_attr_ro(_name) \ -static struct freq_attr _name##_gov_pol = \ -__ATTR(_name, 0444, show_##_name##_gov_pol, NULL) - -#define gov_pol_attr_rw(_name) \ -static struct freq_attr _name##_gov_pol = \ -__ATTR(_name, 0644, show_##_name##_gov_pol, store_##_name##_gov_pol) - -#define gov_sys_pol_attr_rw(_name) \ - gov_sys_attr_rw(_name); \ - gov_pol_attr_rw(_name) +/* Governor demand based switching data (per-policy or global). */ +struct dbs_data { + struct gov_attr_set attr_set; + struct dbs_governor *gov; + void *tuners; + unsigned int ignore_nice_load; + unsigned int sampling_rate; + unsigned int sampling_down_factor; + unsigned int up_threshold; + unsigned int io_is_busy; +}; -#define gov_sys_pol_attr_ro(_name) \ - gov_sys_attr_ro(_name); \ - gov_pol_attr_ro(_name) +static inline struct dbs_data *to_dbs_data(struct gov_attr_set *attr_set) +{ + return container_of(attr_set, struct dbs_data, attr_set); +} -/* Create show/store routines */ -#define show_one(_gov, file_name) \ -static ssize_t show_##file_name##_gov_sys \ -(struct kobject *kobj, struct attribute *attr, char *buf) \ -{ \ - struct _gov##_dbs_tuners *tuners = _gov##_dbs_cdata.gdbs_data->tuners; \ - return sprintf(buf, "%u\n", tuners->file_name); \ -} \ - \ -static ssize_t show_##file_name##_gov_pol \ -(struct cpufreq_policy *policy, char *buf) \ +#define gov_show_one(_gov, file_name) \ +static ssize_t file_name##_show \ +(struct gov_attr_set *attr_set, char *buf) \ { \ - struct dbs_data *dbs_data = policy->governor_data; \ + struct dbs_data *dbs_data = to_dbs_data(attr_set); \ struct _gov##_dbs_tuners *tuners = dbs_data->tuners; \ return sprintf(buf, "%u\n", tuners->file_name); \ } -#define store_one(_gov, file_name) \ -static ssize_t store_##file_name##_gov_sys \ -(struct kobject *kobj, struct attribute *attr, const char *buf, size_t count) \ -{ \ - struct dbs_data *dbs_data = _gov##_dbs_cdata.gdbs_data; \ - return store_##file_name(dbs_data, buf, count); \ -} \ - \ -static ssize_t store_##file_name##_gov_pol \ -(struct cpufreq_policy *policy, const char *buf, size_t count) \ +#define gov_show_one_common(file_name) \ +static ssize_t file_name##_show \ +(struct gov_attr_set *attr_set, char *buf) \ { \ - struct dbs_data *dbs_data = policy->governor_data; \ - return store_##file_name(dbs_data, buf, count); \ + struct dbs_data *dbs_data = to_dbs_data(attr_set); \ + return sprintf(buf, "%u\n", dbs_data->file_name); \ } -#define show_store_one(_gov, file_name) \ -show_one(_gov, file_name); \ -store_one(_gov, file_name) - -/* create helper routines */ -#define define_get_cpu_dbs_routines(_dbs_info) \ -static struct cpu_dbs_common_info *get_cpu_cdbs(int cpu) \ -{ \ - return &per_cpu(_dbs_info, cpu).cdbs; \ -} \ - \ -static void *get_cpu_dbs_info_s(int cpu) \ -{ \ - return &per_cpu(_dbs_info, cpu); \ -} +#define gov_attr_ro(_name) \ +static struct governor_attr _name = __ATTR_RO(_name) -/* - * Abbreviations: - * dbs: used as a shortform for demand based switching It helps to keep variable - * names smaller, simpler - * cdbs: common dbs - * od_*: On-demand governor - * cs_*: Conservative governor - */ +#define gov_attr_rw(_name) \ +static struct governor_attr _name = __ATTR_RW(_name) -/* Per cpu structures */ -struct cpu_dbs_common_info { - int cpu; - u64 prev_cpu_idle; - u64 prev_cpu_wall; - u64 prev_cpu_nice; - struct cpufreq_policy *cur_policy; - struct delayed_work work; +/* Common to all CPUs of a policy */ +struct policy_dbs_info { + struct cpufreq_policy *policy; /* - * percpu mutex that serializes governor limit change with gov_dbs_timer - * invocation. We do not want gov_dbs_timer to run when user is changing - * the governor or limits. + * Per policy mutex that serializes load evaluation from limit-change + * and work-handler. */ - struct mutex timer_mutex; - ktime_t time_stamp; -}; - -struct od_cpu_dbs_info_s { - struct cpu_dbs_common_info cdbs; - struct cpufreq_frequency_table *freq_table; - unsigned int freq_lo; - unsigned int freq_lo_jiffies; - unsigned int freq_hi_jiffies; + struct mutex update_mutex; + + u64 last_sample_time; + s64 sample_delay_ns; + atomic_t work_count; + struct irq_work irq_work; + struct work_struct work; + /* dbs_data may be shared between multiple policy objects */ + struct dbs_data *dbs_data; + struct list_head list; + /* Multiplier for increasing sample delay temporarily. */ unsigned int rate_mult; - unsigned int sample_type:1; -}; - -struct cs_cpu_dbs_info_s { - struct cpu_dbs_common_info cdbs; - unsigned int down_skip; - unsigned int requested_freq; - unsigned int enable:1; + unsigned int idle_periods; /* For conservative */ + /* Status indicators */ + bool is_shared; /* This object is used by multiple CPUs */ + bool work_in_progress; /* Work is being queued up or in progress */ }; -/* Per policy Governers sysfs tunables */ -struct od_dbs_tuners { - unsigned int ignore_nice; - unsigned int sampling_rate; - unsigned int sampling_down_factor; - unsigned int up_threshold; - unsigned int adj_up_threshold; - unsigned int powersave_bias; - unsigned int io_is_busy; -}; +static inline void gov_update_sample_delay(struct policy_dbs_info *policy_dbs, + unsigned int delay_us) +{ + policy_dbs->sample_delay_ns = delay_us * NSEC_PER_USEC; +} -struct cs_dbs_tuners { - unsigned int ignore_nice; - unsigned int sampling_rate; - unsigned int sampling_down_factor; - unsigned int up_threshold; - unsigned int down_threshold; - unsigned int freq_step; +/* Per cpu structures */ +struct cpu_dbs_info { + u64 prev_cpu_idle; + u64 prev_update_time; + u64 prev_cpu_nice; + /* + * Used to keep track of load in the previous interval. However, when + * explicitly set to zero, it is used as a flag to ensure that we copy + * the previous load to the current interval only once, upon the first + * wake-up from idle. + */ + unsigned int prev_load; + struct update_util_data update_util; + struct policy_dbs_info *policy_dbs; }; -/* Common Governer data across policies */ -struct dbs_data; -struct common_dbs_data { - /* Common across governors */ - #define GOV_ONDEMAND 0 - #define GOV_CONSERVATIVE 1 - int governor; - struct attribute_group *attr_group_gov_sys; /* one governor - system */ - struct attribute_group *attr_group_gov_pol; /* one governor - policy */ +/* Common Governor data across policies */ +struct dbs_governor { + struct cpufreq_governor gov; + struct kobj_type kobj_type; - /* Common data for platforms that don't set have_governor_per_policy */ + /* + * Common data for platforms that don't set + * CPUFREQ_HAVE_GOVERNOR_PER_POLICY + */ struct dbs_data *gdbs_data; - struct cpu_dbs_common_info *(*get_cpu_cdbs)(int cpu); - void *(*get_cpu_dbs_info_s)(int cpu); - void (*gov_dbs_timer)(struct work_struct *work); - void (*gov_check_cpu)(int cpu, unsigned int load); + unsigned int (*gov_dbs_update)(struct cpufreq_policy *policy); + struct policy_dbs_info *(*alloc)(void); + void (*free)(struct policy_dbs_info *policy_dbs); int (*init)(struct dbs_data *dbs_data); void (*exit)(struct dbs_data *dbs_data); - - /* Governor specific ops, see below */ - void *gov_ops; + void (*start)(struct cpufreq_policy *policy); }; -/* Governer Per policy data */ -struct dbs_data { - struct common_dbs_data *cdata; - unsigned int min_sampling_rate; - int usage_count; - void *tuners; - - /* dbs_mutex protects dbs_enable in governor start/stop */ - struct mutex mutex; -}; +static inline struct dbs_governor *dbs_governor_of(struct cpufreq_policy *policy) +{ + return container_of(policy->governor, struct dbs_governor, gov); +} -/* Governor specific ops, will be passed to dbs_data->gov_ops */ +/* Governor callback routines */ +int cpufreq_dbs_governor_init(struct cpufreq_policy *policy); +void cpufreq_dbs_governor_exit(struct cpufreq_policy *policy); +int cpufreq_dbs_governor_start(struct cpufreq_policy *policy); +void cpufreq_dbs_governor_stop(struct cpufreq_policy *policy); +void cpufreq_dbs_governor_limits(struct cpufreq_policy *policy); + +#define CPUFREQ_DBS_GOVERNOR_INITIALIZER(_name_) \ + { \ + .name = _name_, \ + .flags = CPUFREQ_GOV_DYNAMIC_SWITCHING, \ + .owner = THIS_MODULE, \ + .init = cpufreq_dbs_governor_init, \ + .exit = cpufreq_dbs_governor_exit, \ + .start = cpufreq_dbs_governor_start, \ + .stop = cpufreq_dbs_governor_stop, \ + .limits = cpufreq_dbs_governor_limits, \ + } + +/* Governor specific operations */ struct od_ops { - void (*powersave_bias_init_cpu)(int cpu); unsigned int (*powersave_bias_target)(struct cpufreq_policy *policy, unsigned int freq_next, unsigned int relation); - void (*freq_increase)(struct cpufreq_policy *p, unsigned int freq); -}; - -struct cs_ops { - struct notifier_block *notifier_block; }; -static inline int delay_for_sampling_rate(unsigned int sampling_rate) -{ - int delay = usecs_to_jiffies(sampling_rate); - - /* We want all CPUs to do sampling nearly on same jiffy */ - if (num_online_cpus() > 1) - delay -= jiffies % delay; - - return delay; -} - -#define declare_show_sampling_rate_min(_gov) \ -static ssize_t show_sampling_rate_min_gov_sys \ -(struct kobject *kobj, struct attribute *attr, char *buf) \ -{ \ - struct dbs_data *dbs_data = _gov##_dbs_cdata.gdbs_data; \ - return sprintf(buf, "%u\n", dbs_data->min_sampling_rate); \ -} \ - \ -static ssize_t show_sampling_rate_min_gov_pol \ -(struct cpufreq_policy *policy, char *buf) \ -{ \ - struct dbs_data *dbs_data = policy->governor_data; \ - return sprintf(buf, "%u\n", dbs_data->min_sampling_rate); \ -} - -void dbs_check_cpu(struct dbs_data *dbs_data, int cpu); -bool need_load_eval(struct cpu_dbs_common_info *cdbs, - unsigned int sampling_rate); -int cpufreq_governor_dbs(struct cpufreq_policy *policy, - struct common_dbs_data *cdata, unsigned int event); -void gov_queue_work(struct dbs_data *dbs_data, struct cpufreq_policy *policy, - unsigned int delay, bool all_cpus); +unsigned int dbs_update(struct cpufreq_policy *policy); void od_register_powersave_bias_handler(unsigned int (*f) (struct cpufreq_policy *, unsigned int, unsigned int), unsigned int powersave_bias); void od_unregister_powersave_bias_handler(void); +ssize_t sampling_rate_store(struct gov_attr_set *attr_set, const char *buf, + size_t count); +void gov_update_cpu_data(struct dbs_data *dbs_data); #endif /* _CPUFREQ_GOVERNOR_H */ diff --git a/drivers/cpufreq/cpufreq_governor_attr_set.c b/drivers/cpufreq/cpufreq_governor_attr_set.c new file mode 100644 index 000000000000..771770ea0ed0 --- /dev/null +++ b/drivers/cpufreq/cpufreq_governor_attr_set.c @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Abstract code for CPUFreq governor tunable sysfs attributes. + * + * Copyright (C) 2016, Intel Corporation + * Author: Rafael J. Wysocki <rafael.j.wysocki@intel.com> + */ + +#include "cpufreq_governor.h" + +static inline struct governor_attr *to_gov_attr(struct attribute *attr) +{ + return container_of(attr, struct governor_attr, attr); +} + +static ssize_t governor_show(struct kobject *kobj, struct attribute *attr, + char *buf) +{ + struct governor_attr *gattr = to_gov_attr(attr); + + return gattr->show(to_gov_attr_set(kobj), buf); +} + +static ssize_t governor_store(struct kobject *kobj, struct attribute *attr, + const char *buf, size_t count) +{ + struct gov_attr_set *attr_set = to_gov_attr_set(kobj); + struct governor_attr *gattr = to_gov_attr(attr); + int ret; + + mutex_lock(&attr_set->update_lock); + ret = attr_set->usage_count ? gattr->store(attr_set, buf, count) : -EBUSY; + mutex_unlock(&attr_set->update_lock); + return ret; +} + +const struct sysfs_ops governor_sysfs_ops = { + .show = governor_show, + .store = governor_store, +}; +EXPORT_SYMBOL_GPL(governor_sysfs_ops); + +void gov_attr_set_init(struct gov_attr_set *attr_set, struct list_head *list_node) +{ + INIT_LIST_HEAD(&attr_set->policy_list); + mutex_init(&attr_set->update_lock); + attr_set->usage_count = 1; + list_add(list_node, &attr_set->policy_list); +} +EXPORT_SYMBOL_GPL(gov_attr_set_init); + +void gov_attr_set_get(struct gov_attr_set *attr_set, struct list_head *list_node) +{ + mutex_lock(&attr_set->update_lock); + attr_set->usage_count++; + list_add(list_node, &attr_set->policy_list); + mutex_unlock(&attr_set->update_lock); +} +EXPORT_SYMBOL_GPL(gov_attr_set_get); + +unsigned int gov_attr_set_put(struct gov_attr_set *attr_set, struct list_head *list_node) +{ + unsigned int count; + + mutex_lock(&attr_set->update_lock); + list_del(list_node); + count = --attr_set->usage_count; + mutex_unlock(&attr_set->update_lock); + if (count) + return count; + + mutex_destroy(&attr_set->update_lock); + kobject_put(&attr_set->kobj); + return 0; +} +EXPORT_SYMBOL_GPL(gov_attr_set_put); diff --git a/drivers/cpufreq/cpufreq_ondemand.c b/drivers/cpufreq/cpufreq_ondemand.c index 93eb5cbcc1f6..a6ecc203f7b7 100644 --- a/drivers/cpufreq/cpufreq_ondemand.c +++ b/drivers/cpufreq/cpufreq_ondemand.c @@ -1,371 +1,206 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * drivers/cpufreq/cpufreq_ondemand.c * * Copyright (C) 2001 Russell King * (C) 2003 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>. * Jun Nakajima <jun.nakajima@intel.com> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -#include <linux/cpufreq.h> -#include <linux/init.h> -#include <linux/kernel.h> -#include <linux/kernel_stat.h> -#include <linux/kobject.h> -#include <linux/module.h> -#include <linux/mutex.h> +#include <linux/cpu.h> #include <linux/percpu-defs.h> #include <linux/slab.h> -#include <linux/sysfs.h> #include <linux/tick.h> -#include <linux/types.h> -#include <linux/cpu.h> +#include <linux/sched/cpufreq.h> -#include "cpufreq_governor.h" +#include "cpufreq_ondemand.h" /* On-demand governor macros */ -#define DEF_FREQUENCY_DOWN_DIFFERENTIAL (10) #define DEF_FREQUENCY_UP_THRESHOLD (80) #define DEF_SAMPLING_DOWN_FACTOR (1) #define MAX_SAMPLING_DOWN_FACTOR (100000) -#define MICRO_FREQUENCY_DOWN_DIFFERENTIAL (3) #define MICRO_FREQUENCY_UP_THRESHOLD (95) -#define MICRO_FREQUENCY_MIN_SAMPLE_RATE (10000) -#define MIN_FREQUENCY_UP_THRESHOLD (11) +#define MIN_FREQUENCY_UP_THRESHOLD (1) #define MAX_FREQUENCY_UP_THRESHOLD (100) -static DEFINE_PER_CPU(struct od_cpu_dbs_info_s, od_cpu_dbs_info); - static struct od_ops od_ops; -#ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND -static struct cpufreq_governor cpufreq_gov_ondemand; -#endif - static unsigned int default_powersave_bias; -static void ondemand_powersave_bias_init_cpu(int cpu) -{ - struct od_cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info, cpu); - - dbs_info->freq_table = cpufreq_frequency_get_table(cpu); - dbs_info->freq_lo = 0; -} - -/* - * Not all CPUs want IO time to be accounted as busy; this depends on how - * efficient idling at a higher frequency/voltage is. - * Pavel Machek says this is not so for various generations of AMD and old - * Intel systems. - * Mike Chan (android.com) claims this is also not true for ARM. - * Because of this, whitelist specific known (series) of CPUs by default, and - * leave all others up to the user. - */ -static int should_io_be_busy(void) -{ -#if defined(CONFIG_X86) - /* - * For Intel, Core 2 (model 15) and later have an efficient idle. - */ - if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL && - boot_cpu_data.x86 == 6 && - boot_cpu_data.x86_model >= 15) - return 1; -#endif - return 0; -} - /* * Find right freq to be set now with powersave_bias on. - * Returns the freq_hi to be used right now and will set freq_hi_jiffies, - * freq_lo, and freq_lo_jiffies in percpu area for averaging freqs. + * Returns the freq_hi to be used right now and will set freq_hi_delay_us, + * freq_lo, and freq_lo_delay_us in percpu area for averaging freqs. */ static unsigned int generic_powersave_bias_target(struct cpufreq_policy *policy, unsigned int freq_next, unsigned int relation) { unsigned int freq_req, freq_reduc, freq_avg; unsigned int freq_hi, freq_lo; - unsigned int index = 0; - unsigned int jiffies_total, jiffies_hi, jiffies_lo; - struct od_cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info, - policy->cpu); - struct dbs_data *dbs_data = policy->governor_data; + unsigned int index; + unsigned int delay_hi_us; + struct policy_dbs_info *policy_dbs = policy->governor_data; + struct od_policy_dbs_info *dbs_info = to_dbs_info(policy_dbs); + struct dbs_data *dbs_data = policy_dbs->dbs_data; struct od_dbs_tuners *od_tuners = dbs_data->tuners; + struct cpufreq_frequency_table *freq_table = policy->freq_table; - if (!dbs_info->freq_table) { + if (!freq_table) { dbs_info->freq_lo = 0; - dbs_info->freq_lo_jiffies = 0; + dbs_info->freq_lo_delay_us = 0; return freq_next; } - cpufreq_frequency_table_target(policy, dbs_info->freq_table, freq_next, - relation, &index); - freq_req = dbs_info->freq_table[index].frequency; + index = cpufreq_frequency_table_target(policy, freq_next, policy->min, + policy->max, relation); + freq_req = freq_table[index].frequency; freq_reduc = freq_req * od_tuners->powersave_bias / 1000; freq_avg = freq_req - freq_reduc; /* Find freq bounds for freq_avg in freq_table */ - index = 0; - cpufreq_frequency_table_target(policy, dbs_info->freq_table, freq_avg, - CPUFREQ_RELATION_H, &index); - freq_lo = dbs_info->freq_table[index].frequency; - index = 0; - cpufreq_frequency_table_target(policy, dbs_info->freq_table, freq_avg, - CPUFREQ_RELATION_L, &index); - freq_hi = dbs_info->freq_table[index].frequency; + index = cpufreq_table_find_index_h(policy, freq_avg, + relation & CPUFREQ_RELATION_E); + freq_lo = freq_table[index].frequency; + index = cpufreq_table_find_index_l(policy, freq_avg, + relation & CPUFREQ_RELATION_E); + freq_hi = freq_table[index].frequency; /* Find out how long we have to be in hi and lo freqs */ if (freq_hi == freq_lo) { dbs_info->freq_lo = 0; - dbs_info->freq_lo_jiffies = 0; + dbs_info->freq_lo_delay_us = 0; return freq_lo; } - jiffies_total = usecs_to_jiffies(od_tuners->sampling_rate); - jiffies_hi = (freq_avg - freq_lo) * jiffies_total; - jiffies_hi += ((freq_hi - freq_lo) / 2); - jiffies_hi /= (freq_hi - freq_lo); - jiffies_lo = jiffies_total - jiffies_hi; + delay_hi_us = (freq_avg - freq_lo) * dbs_data->sampling_rate; + delay_hi_us += (freq_hi - freq_lo) / 2; + delay_hi_us /= freq_hi - freq_lo; + dbs_info->freq_hi_delay_us = delay_hi_us; dbs_info->freq_lo = freq_lo; - dbs_info->freq_lo_jiffies = jiffies_lo; - dbs_info->freq_hi_jiffies = jiffies_hi; + dbs_info->freq_lo_delay_us = dbs_data->sampling_rate - delay_hi_us; return freq_hi; } -static void ondemand_powersave_bias_init(void) +static void ondemand_powersave_bias_init(struct cpufreq_policy *policy) { - int i; - for_each_online_cpu(i) { - ondemand_powersave_bias_init_cpu(i); - } + struct od_policy_dbs_info *dbs_info = to_dbs_info(policy->governor_data); + + dbs_info->freq_lo = 0; } -static void dbs_freq_increase(struct cpufreq_policy *p, unsigned int freq) +static void dbs_freq_increase(struct cpufreq_policy *policy, unsigned int freq) { - struct dbs_data *dbs_data = p->governor_data; + struct policy_dbs_info *policy_dbs = policy->governor_data; + struct dbs_data *dbs_data = policy_dbs->dbs_data; struct od_dbs_tuners *od_tuners = dbs_data->tuners; if (od_tuners->powersave_bias) - freq = od_ops.powersave_bias_target(p, freq, - CPUFREQ_RELATION_H); - else if (p->cur == p->max) + freq = od_ops.powersave_bias_target(policy, freq, + CPUFREQ_RELATION_HE); + else if (policy->cur == policy->max) return; - __cpufreq_driver_target(p, freq, od_tuners->powersave_bias ? - CPUFREQ_RELATION_L : CPUFREQ_RELATION_H); + __cpufreq_driver_target(policy, freq, od_tuners->powersave_bias ? + CPUFREQ_RELATION_LE : CPUFREQ_RELATION_HE); } /* * Every sampling_rate, we check, if current idle time is less than 20% - * (default), then we try to increase frequency. Every sampling_rate, we look - * for the lowest frequency which can sustain the load while keeping idle time - * over 30%. If such a frequency exist, we try to decrease to this frequency. - * - * Any frequency increase takes it to the maximum frequency. Frequency reduction - * happens at minimum steps of 5% (default) of current frequency + * (default), then we try to increase frequency. Else, we adjust the frequency + * proportional to load. */ -static void od_check_cpu(int cpu, unsigned int load_freq) +static void od_update(struct cpufreq_policy *policy) { - struct od_cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info, cpu); - struct cpufreq_policy *policy = dbs_info->cdbs.cur_policy; - struct dbs_data *dbs_data = policy->governor_data; + struct policy_dbs_info *policy_dbs = policy->governor_data; + struct od_policy_dbs_info *dbs_info = to_dbs_info(policy_dbs); + struct dbs_data *dbs_data = policy_dbs->dbs_data; struct od_dbs_tuners *od_tuners = dbs_data->tuners; + unsigned int load = dbs_update(policy); dbs_info->freq_lo = 0; /* Check for frequency increase */ - if (load_freq > od_tuners->up_threshold * policy->cur) { + if (load > dbs_data->up_threshold) { /* If switching to max speed, apply sampling_down_factor */ if (policy->cur < policy->max) - dbs_info->rate_mult = - od_tuners->sampling_down_factor; + policy_dbs->rate_mult = dbs_data->sampling_down_factor; dbs_freq_increase(policy, policy->max); - return; - } - - /* Check for frequency decrease */ - /* if we cannot reduce the frequency anymore, break out early */ - if (policy->cur == policy->min) - return; + } else { + /* Calculate the next frequency proportional to load */ + unsigned int freq_next, min_f, max_f; - /* - * The optimal frequency is the frequency that is the lowest that can - * support the current CPU usage without triggering the up policy. To be - * safe, we focus 10 points under the threshold. - */ - if (load_freq < od_tuners->adj_up_threshold - * policy->cur) { - unsigned int freq_next; - freq_next = load_freq / od_tuners->adj_up_threshold; + min_f = policy->cpuinfo.min_freq; + max_f = policy->cpuinfo.max_freq; + freq_next = min_f + load * (max_f - min_f) / 100; /* No longer fully busy, reset rate_mult */ - dbs_info->rate_mult = 1; - - if (freq_next < policy->min) - freq_next = policy->min; + policy_dbs->rate_mult = 1; - if (!od_tuners->powersave_bias) { - __cpufreq_driver_target(policy, freq_next, - CPUFREQ_RELATION_L); - return; - } + if (od_tuners->powersave_bias) + freq_next = od_ops.powersave_bias_target(policy, + freq_next, + CPUFREQ_RELATION_LE); - freq_next = od_ops.powersave_bias_target(policy, freq_next, - CPUFREQ_RELATION_L); - __cpufreq_driver_target(policy, freq_next, CPUFREQ_RELATION_L); + __cpufreq_driver_target(policy, freq_next, CPUFREQ_RELATION_CE); } } -static void od_dbs_timer(struct work_struct *work) +static unsigned int od_dbs_update(struct cpufreq_policy *policy) { - struct od_cpu_dbs_info_s *dbs_info = - container_of(work, struct od_cpu_dbs_info_s, cdbs.work.work); - unsigned int cpu = dbs_info->cdbs.cur_policy->cpu; - struct od_cpu_dbs_info_s *core_dbs_info = &per_cpu(od_cpu_dbs_info, - cpu); - struct dbs_data *dbs_data = dbs_info->cdbs.cur_policy->governor_data; - struct od_dbs_tuners *od_tuners = dbs_data->tuners; - int delay = 0, sample_type = core_dbs_info->sample_type; - bool modify_all = true; - - mutex_lock(&core_dbs_info->cdbs.timer_mutex); - if (!need_load_eval(&core_dbs_info->cdbs, od_tuners->sampling_rate)) { - modify_all = false; - goto max_delay; - } + struct policy_dbs_info *policy_dbs = policy->governor_data; + struct dbs_data *dbs_data = policy_dbs->dbs_data; + struct od_policy_dbs_info *dbs_info = to_dbs_info(policy_dbs); + int sample_type = dbs_info->sample_type; /* Common NORMAL_SAMPLE setup */ - core_dbs_info->sample_type = OD_NORMAL_SAMPLE; - if (sample_type == OD_SUB_SAMPLE) { - delay = core_dbs_info->freq_lo_jiffies; - __cpufreq_driver_target(core_dbs_info->cdbs.cur_policy, - core_dbs_info->freq_lo, CPUFREQ_RELATION_H); - } else { - dbs_check_cpu(dbs_data, cpu); - if (core_dbs_info->freq_lo) { - /* Setup timer for SUB_SAMPLE */ - core_dbs_info->sample_type = OD_SUB_SAMPLE; - delay = core_dbs_info->freq_hi_jiffies; - } + dbs_info->sample_type = OD_NORMAL_SAMPLE; + /* + * OD_SUB_SAMPLE doesn't make sense if sample_delay_ns is 0, so ignore + * it then. + */ + if (sample_type == OD_SUB_SAMPLE && policy_dbs->sample_delay_ns > 0) { + __cpufreq_driver_target(policy, dbs_info->freq_lo, + CPUFREQ_RELATION_HE); + return dbs_info->freq_lo_delay_us; } -max_delay: - if (!delay) - delay = delay_for_sampling_rate(od_tuners->sampling_rate - * core_dbs_info->rate_mult); - - gov_queue_work(dbs_data, dbs_info->cdbs.cur_policy, delay, modify_all); - mutex_unlock(&core_dbs_info->cdbs.timer_mutex); -} - -/************************** sysfs interface ************************/ -static struct common_dbs_data od_dbs_cdata; - -/** - * update_sampling_rate - update sampling rate effective immediately if needed. - * @new_rate: new sampling rate - * - * If new rate is smaller than the old, simply updating - * dbs_tuners_int.sampling_rate might not be appropriate. For example, if the - * original sampling_rate was 1 second and the requested new sampling rate is 10 - * ms because the user needs immediate reaction from ondemand governor, but not - * sure if higher frequency will be required or not, then, the governor may - * change the sampling rate too late; up to 1 second later. Thus, if we are - * reducing the sampling rate, we need to make the new value effective - * immediately. - */ -static void update_sampling_rate(struct dbs_data *dbs_data, - unsigned int new_rate) -{ - struct od_dbs_tuners *od_tuners = dbs_data->tuners; - int cpu; + od_update(policy); - od_tuners->sampling_rate = new_rate = max(new_rate, - dbs_data->min_sampling_rate); - - for_each_online_cpu(cpu) { - struct cpufreq_policy *policy; - struct od_cpu_dbs_info_s *dbs_info; - unsigned long next_sampling, appointed_at; - - policy = cpufreq_cpu_get(cpu); - if (!policy) - continue; - if (policy->governor != &cpufreq_gov_ondemand) { - cpufreq_cpu_put(policy); - continue; - } - dbs_info = &per_cpu(od_cpu_dbs_info, cpu); - cpufreq_cpu_put(policy); - - mutex_lock(&dbs_info->cdbs.timer_mutex); - - if (!delayed_work_pending(&dbs_info->cdbs.work)) { - mutex_unlock(&dbs_info->cdbs.timer_mutex); - continue; - } - - next_sampling = jiffies + usecs_to_jiffies(new_rate); - appointed_at = dbs_info->cdbs.work.timer.expires; - - if (time_before(next_sampling, appointed_at)) { - - mutex_unlock(&dbs_info->cdbs.timer_mutex); - cancel_delayed_work_sync(&dbs_info->cdbs.work); - mutex_lock(&dbs_info->cdbs.timer_mutex); - - gov_queue_work(dbs_data, dbs_info->cdbs.cur_policy, - usecs_to_jiffies(new_rate), true); - - } - mutex_unlock(&dbs_info->cdbs.timer_mutex); + if (dbs_info->freq_lo) { + /* Setup SUB_SAMPLE */ + dbs_info->sample_type = OD_SUB_SAMPLE; + return dbs_info->freq_hi_delay_us; } -} -static ssize_t store_sampling_rate(struct dbs_data *dbs_data, const char *buf, - size_t count) -{ - unsigned int input; - int ret; - ret = sscanf(buf, "%u", &input); - if (ret != 1) - return -EINVAL; - - update_sampling_rate(dbs_data, input); - return count; + return dbs_data->sampling_rate * policy_dbs->rate_mult; } -static ssize_t store_io_is_busy(struct dbs_data *dbs_data, const char *buf, - size_t count) +/************************** sysfs interface ************************/ +static struct dbs_governor od_dbs_gov; + +static ssize_t io_is_busy_store(struct gov_attr_set *attr_set, const char *buf, + size_t count) { - struct od_dbs_tuners *od_tuners = dbs_data->tuners; + struct dbs_data *dbs_data = to_dbs_data(attr_set); unsigned int input; int ret; - unsigned int j; ret = sscanf(buf, "%u", &input); if (ret != 1) return -EINVAL; - od_tuners->io_is_busy = !!input; + dbs_data->io_is_busy = !!input; /* we need to re-evaluate prev_cpu_idle */ - for_each_online_cpu(j) { - struct od_cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info, - j); - dbs_info->cdbs.prev_cpu_idle = get_cpu_idle_time(j, - &dbs_info->cdbs.prev_cpu_wall, od_tuners->io_is_busy); - } + gov_update_cpu_data(dbs_data); + return count; } -static ssize_t store_up_threshold(struct dbs_data *dbs_data, const char *buf, - size_t count) +static ssize_t up_threshold_store(struct gov_attr_set *attr_set, + const char *buf, size_t count) { - struct od_dbs_tuners *od_tuners = dbs_data->tuners; + struct dbs_data *dbs_data = to_dbs_data(attr_set); unsigned int input; int ret; ret = sscanf(buf, "%u", &input); @@ -374,44 +209,46 @@ static ssize_t store_up_threshold(struct dbs_data *dbs_data, const char *buf, input < MIN_FREQUENCY_UP_THRESHOLD) { return -EINVAL; } - /* Calculate the new adj_up_threshold */ - od_tuners->adj_up_threshold += input; - od_tuners->adj_up_threshold -= od_tuners->up_threshold; - od_tuners->up_threshold = input; + dbs_data->up_threshold = input; return count; } -static ssize_t store_sampling_down_factor(struct dbs_data *dbs_data, - const char *buf, size_t count) +static ssize_t sampling_down_factor_store(struct gov_attr_set *attr_set, + const char *buf, size_t count) { - struct od_dbs_tuners *od_tuners = dbs_data->tuners; - unsigned int input, j; + struct dbs_data *dbs_data = to_dbs_data(attr_set); + struct policy_dbs_info *policy_dbs; + unsigned int input; int ret; ret = sscanf(buf, "%u", &input); if (ret != 1 || input > MAX_SAMPLING_DOWN_FACTOR || input < 1) return -EINVAL; - od_tuners->sampling_down_factor = input; + + dbs_data->sampling_down_factor = input; /* Reset down sampling multiplier in case it was active */ - for_each_online_cpu(j) { - struct od_cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info, - j); - dbs_info->rate_mult = 1; + list_for_each_entry(policy_dbs, &attr_set->policy_list, list) { + /* + * Doing this without locking might lead to using different + * rate_mult values in od_update() and od_dbs_update(). + */ + mutex_lock(&policy_dbs->update_mutex); + policy_dbs->rate_mult = 1; + mutex_unlock(&policy_dbs->update_mutex); } + return count; } -static ssize_t store_ignore_nice(struct dbs_data *dbs_data, const char *buf, - size_t count) +static ssize_t ignore_nice_load_store(struct gov_attr_set *attr_set, + const char *buf, size_t count) { - struct od_dbs_tuners *od_tuners = dbs_data->tuners; + struct dbs_data *dbs_data = to_dbs_data(attr_set); unsigned int input; int ret; - unsigned int j; - ret = sscanf(buf, "%u", &input); if (ret != 1) return -EINVAL; @@ -419,29 +256,23 @@ static ssize_t store_ignore_nice(struct dbs_data *dbs_data, const char *buf, if (input > 1) input = 1; - if (input == od_tuners->ignore_nice) { /* nothing to do */ + if (input == dbs_data->ignore_nice_load) { /* nothing to do */ return count; } - od_tuners->ignore_nice = input; + dbs_data->ignore_nice_load = input; /* we need to re-evaluate prev_cpu_idle */ - for_each_online_cpu(j) { - struct od_cpu_dbs_info_s *dbs_info; - dbs_info = &per_cpu(od_cpu_dbs_info, j); - dbs_info->cdbs.prev_cpu_idle = get_cpu_idle_time(j, - &dbs_info->cdbs.prev_cpu_wall, od_tuners->io_is_busy); - if (od_tuners->ignore_nice) - dbs_info->cdbs.prev_cpu_nice = - kcpustat_cpu(j).cpustat[CPUTIME_NICE]; + gov_update_cpu_data(dbs_data); - } return count; } -static ssize_t store_powersave_bias(struct dbs_data *dbs_data, const char *buf, - size_t count) +static ssize_t powersave_bias_store(struct gov_attr_set *attr_set, + const char *buf, size_t count) { + struct dbs_data *dbs_data = to_dbs_data(attr_set); struct od_dbs_tuners *od_tuners = dbs_data->tuners; + struct policy_dbs_info *policy_dbs; unsigned int input; int ret; ret = sscanf(buf, "%u", &input); @@ -453,59 +284,52 @@ static ssize_t store_powersave_bias(struct dbs_data *dbs_data, const char *buf, input = 1000; od_tuners->powersave_bias = input; - ondemand_powersave_bias_init(); + + list_for_each_entry(policy_dbs, &attr_set->policy_list, list) + ondemand_powersave_bias_init(policy_dbs->policy); + return count; } -show_store_one(od, sampling_rate); -show_store_one(od, io_is_busy); -show_store_one(od, up_threshold); -show_store_one(od, sampling_down_factor); -show_store_one(od, ignore_nice); -show_store_one(od, powersave_bias); -declare_show_sampling_rate_min(od); - -gov_sys_pol_attr_rw(sampling_rate); -gov_sys_pol_attr_rw(io_is_busy); -gov_sys_pol_attr_rw(up_threshold); -gov_sys_pol_attr_rw(sampling_down_factor); -gov_sys_pol_attr_rw(ignore_nice); -gov_sys_pol_attr_rw(powersave_bias); -gov_sys_pol_attr_ro(sampling_rate_min); - -static struct attribute *dbs_attributes_gov_sys[] = { - &sampling_rate_min_gov_sys.attr, - &sampling_rate_gov_sys.attr, - &up_threshold_gov_sys.attr, - &sampling_down_factor_gov_sys.attr, - &ignore_nice_gov_sys.attr, - &powersave_bias_gov_sys.attr, - &io_is_busy_gov_sys.attr, +gov_show_one_common(sampling_rate); +gov_show_one_common(up_threshold); +gov_show_one_common(sampling_down_factor); +gov_show_one_common(ignore_nice_load); +gov_show_one_common(io_is_busy); +gov_show_one(od, powersave_bias); + +gov_attr_rw(sampling_rate); +gov_attr_rw(io_is_busy); +gov_attr_rw(up_threshold); +gov_attr_rw(sampling_down_factor); +gov_attr_rw(ignore_nice_load); +gov_attr_rw(powersave_bias); + +static struct attribute *od_attrs[] = { + &sampling_rate.attr, + &up_threshold.attr, + &sampling_down_factor.attr, + &ignore_nice_load.attr, + &powersave_bias.attr, + &io_is_busy.attr, NULL }; +ATTRIBUTE_GROUPS(od); -static struct attribute_group od_attr_group_gov_sys = { - .attrs = dbs_attributes_gov_sys, - .name = "ondemand", -}; +/************************** sysfs end ************************/ -static struct attribute *dbs_attributes_gov_pol[] = { - &sampling_rate_min_gov_pol.attr, - &sampling_rate_gov_pol.attr, - &up_threshold_gov_pol.attr, - &sampling_down_factor_gov_pol.attr, - &ignore_nice_gov_pol.attr, - &powersave_bias_gov_pol.attr, - &io_is_busy_gov_pol.attr, - NULL -}; +static struct policy_dbs_info *od_alloc(void) +{ + struct od_policy_dbs_info *dbs_info; -static struct attribute_group od_attr_group_gov_pol = { - .attrs = dbs_attributes_gov_pol, - .name = "ondemand", -}; + dbs_info = kzalloc(sizeof(*dbs_info), GFP_KERNEL); + return dbs_info ? &dbs_info->policy_dbs : NULL; +} -/************************** sysfs end ************************/ +static void od_free(struct policy_dbs_info *policy_dbs) +{ + kfree(to_dbs_info(policy_dbs)); +} static int od_init(struct dbs_data *dbs_data) { @@ -513,43 +337,26 @@ static int od_init(struct dbs_data *dbs_data) u64 idle_time; int cpu; - tuners = kzalloc(sizeof(struct od_dbs_tuners), GFP_KERNEL); - if (!tuners) { - pr_err("%s: kzalloc failed\n", __func__); + tuners = kzalloc(sizeof(*tuners), GFP_KERNEL); + if (!tuners) return -ENOMEM; - } cpu = get_cpu(); idle_time = get_cpu_idle_time_us(cpu, NULL); put_cpu(); if (idle_time != -1ULL) { /* Idle micro accounting is supported. Use finer thresholds */ - tuners->up_threshold = MICRO_FREQUENCY_UP_THRESHOLD; - tuners->adj_up_threshold = MICRO_FREQUENCY_UP_THRESHOLD - - MICRO_FREQUENCY_DOWN_DIFFERENTIAL; - /* - * In nohz/micro accounting case we set the minimum frequency - * not depending on HZ, but fixed (very low). The deferred - * timer might skip some samples if idle/sleeping as needed. - */ - dbs_data->min_sampling_rate = MICRO_FREQUENCY_MIN_SAMPLE_RATE; + dbs_data->up_threshold = MICRO_FREQUENCY_UP_THRESHOLD; } else { - tuners->up_threshold = DEF_FREQUENCY_UP_THRESHOLD; - tuners->adj_up_threshold = DEF_FREQUENCY_UP_THRESHOLD - - DEF_FREQUENCY_DOWN_DIFFERENTIAL; - - /* For correct statistics, we need 10 ticks for each measure */ - dbs_data->min_sampling_rate = MIN_SAMPLING_RATE_RATIO * - jiffies_to_usecs(10); + dbs_data->up_threshold = DEF_FREQUENCY_UP_THRESHOLD; } - tuners->sampling_down_factor = DEF_SAMPLING_DOWN_FACTOR; - tuners->ignore_nice = 0; + dbs_data->sampling_down_factor = DEF_SAMPLING_DOWN_FACTOR; + dbs_data->ignore_nice_load = 0; tuners->powersave_bias = default_powersave_bias; - tuners->io_is_busy = should_io_be_busy(); + dbs_data->io_is_busy = od_should_io_be_busy(); dbs_data->tuners = tuners; - mutex_init(&dbs_data->mutex); return 0; } @@ -558,57 +365,69 @@ static void od_exit(struct dbs_data *dbs_data) kfree(dbs_data->tuners); } -define_get_cpu_dbs_routines(od_cpu_dbs_info); +static void od_start(struct cpufreq_policy *policy) +{ + struct od_policy_dbs_info *dbs_info = to_dbs_info(policy->governor_data); + + dbs_info->sample_type = OD_NORMAL_SAMPLE; + ondemand_powersave_bias_init(policy); +} static struct od_ops od_ops = { - .powersave_bias_init_cpu = ondemand_powersave_bias_init_cpu, .powersave_bias_target = generic_powersave_bias_target, - .freq_increase = dbs_freq_increase, }; -static struct common_dbs_data od_dbs_cdata = { - .governor = GOV_ONDEMAND, - .attr_group_gov_sys = &od_attr_group_gov_sys, - .attr_group_gov_pol = &od_attr_group_gov_pol, - .get_cpu_cdbs = get_cpu_cdbs, - .get_cpu_dbs_info_s = get_cpu_dbs_info_s, - .gov_dbs_timer = od_dbs_timer, - .gov_check_cpu = od_check_cpu, - .gov_ops = &od_ops, +static struct dbs_governor od_dbs_gov = { + .gov = CPUFREQ_DBS_GOVERNOR_INITIALIZER("ondemand"), + .kobj_type = { .default_groups = od_groups }, + .gov_dbs_update = od_dbs_update, + .alloc = od_alloc, + .free = od_free, .init = od_init, .exit = od_exit, + .start = od_start, }; +#define CPU_FREQ_GOV_ONDEMAND (od_dbs_gov.gov) + static void od_set_powersave_bias(unsigned int powersave_bias) { - struct cpufreq_policy *policy; - struct dbs_data *dbs_data; - struct od_dbs_tuners *od_tuners; unsigned int cpu; - cpumask_t done; + cpumask_var_t done; + + if (!alloc_cpumask_var(&done, GFP_KERNEL)) + return; default_powersave_bias = powersave_bias; - cpumask_clear(&done); + cpumask_clear(done); - get_online_cpus(); + cpus_read_lock(); for_each_online_cpu(cpu) { - if (cpumask_test_cpu(cpu, &done)) - continue; + struct cpufreq_policy *policy; + struct policy_dbs_info *policy_dbs; + struct dbs_data *dbs_data; + struct od_dbs_tuners *od_tuners; - policy = per_cpu(od_cpu_dbs_info, cpu).cdbs.cur_policy; - if (!policy) + if (cpumask_test_cpu(cpu, done)) continue; - cpumask_or(&done, &done, policy->cpus); + policy = cpufreq_cpu_get_raw(cpu); + if (!policy || policy->governor != &CPU_FREQ_GOV_ONDEMAND) + continue; - if (policy->governor != &cpufreq_gov_ondemand) + policy_dbs = policy->governor_data; + if (!policy_dbs) continue; - dbs_data = policy->governor_data; + cpumask_or(done, done, policy->cpus); + + dbs_data = policy_dbs->dbs_data; od_tuners = dbs_data->tuners; od_tuners->powersave_bias = default_powersave_bias; } - put_online_cpus(); + cpus_read_unlock(); + + free_cpumask_var(done); } void od_register_powersave_bias_handler(unsigned int (*f) @@ -627,32 +446,6 @@ void od_unregister_powersave_bias_handler(void) } EXPORT_SYMBOL_GPL(od_unregister_powersave_bias_handler); -static int od_cpufreq_governor_dbs(struct cpufreq_policy *policy, - unsigned int event) -{ - return cpufreq_governor_dbs(policy, &od_dbs_cdata, event); -} - -#ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND -static -#endif -struct cpufreq_governor cpufreq_gov_ondemand = { - .name = "ondemand", - .governor = od_cpufreq_governor_dbs, - .max_transition_latency = TRANSITION_LATENCY_LIMIT, - .owner = THIS_MODULE, -}; - -static int __init cpufreq_gov_dbs_init(void) -{ - return cpufreq_register_governor(&cpufreq_gov_ondemand); -} - -static void __exit cpufreq_gov_dbs_exit(void) -{ - cpufreq_unregister_governor(&cpufreq_gov_ondemand); -} - MODULE_AUTHOR("Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>"); MODULE_AUTHOR("Alexey Starikovskiy <alexey.y.starikovskiy@intel.com>"); MODULE_DESCRIPTION("'cpufreq_ondemand' - A dynamic cpufreq governor for " @@ -660,8 +453,11 @@ MODULE_DESCRIPTION("'cpufreq_ondemand' - A dynamic cpufreq governor for " MODULE_LICENSE("GPL"); #ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND -fs_initcall(cpufreq_gov_dbs_init); -#else -module_init(cpufreq_gov_dbs_init); +struct cpufreq_governor *cpufreq_default_governor(void) +{ + return &CPU_FREQ_GOV_ONDEMAND; +} #endif -module_exit(cpufreq_gov_dbs_exit); + +cpufreq_governor_init(CPU_FREQ_GOV_ONDEMAND); +cpufreq_governor_exit(CPU_FREQ_GOV_ONDEMAND); diff --git a/drivers/cpufreq/cpufreq_ondemand.h b/drivers/cpufreq/cpufreq_ondemand.h new file mode 100644 index 000000000000..2ca8f1aaf2e3 --- /dev/null +++ b/drivers/cpufreq/cpufreq_ondemand.h @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Header file for CPUFreq ondemand governor and related code. + * + * Copyright (C) 2016, Intel Corporation + * Author: Rafael J. Wysocki <rafael.j.wysocki@intel.com> + */ + +#include "cpufreq_governor.h" + +struct od_policy_dbs_info { + struct policy_dbs_info policy_dbs; + unsigned int freq_lo; + unsigned int freq_lo_delay_us; + unsigned int freq_hi_delay_us; + unsigned int sample_type:1; +}; + +static inline struct od_policy_dbs_info *to_dbs_info(struct policy_dbs_info *policy_dbs) +{ + return container_of(policy_dbs, struct od_policy_dbs_info, policy_dbs); +} + +struct od_dbs_tuners { + unsigned int powersave_bias; +}; + +#ifdef CONFIG_X86 +#include <asm/cpu_device_id.h> + +/* + * Not all CPUs want IO time to be accounted as busy; this depends on + * how efficient idling at a higher frequency/voltage is. + * + * Pavel Machek says this is not so for various generations of AMD and + * old Intel systems. Mike Chan (android.com) claims this is also not + * true for ARM. + * + * Because of this, select a known series of Intel CPUs (Family 6 and + * later) by default, and leave all others up to the user. + */ +static inline bool od_should_io_be_busy(void) +{ + return (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL && + boot_cpu_data.x86_vfm >= INTEL_PENTIUM_PRO); +} +#else +static inline bool od_should_io_be_busy(void) { return false; } +#endif diff --git a/drivers/cpufreq/cpufreq_performance.c b/drivers/cpufreq/cpufreq_performance.c index 9fef7d6e4e6a..addd93f2a420 100644 --- a/drivers/cpufreq/cpufreq_performance.c +++ b/drivers/cpufreq/cpufreq_performance.c @@ -1,61 +1,45 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * linux/drivers/cpufreq/cpufreq_performance.c * * Copyright (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de> - * - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -#include <linux/kernel.h> -#include <linux/module.h> #include <linux/cpufreq.h> #include <linux/init.h> +#include <linux/module.h> -static int cpufreq_governor_performance(struct cpufreq_policy *policy, - unsigned int event) +static void cpufreq_gov_performance_limits(struct cpufreq_policy *policy) { - switch (event) { - case CPUFREQ_GOV_START: - case CPUFREQ_GOV_LIMITS: - pr_debug("setting to %u kHz because of event %u\n", - policy->max, event); - __cpufreq_driver_target(policy, policy->max, - CPUFREQ_RELATION_H); - break; - default: - break; - } - return 0; + pr_debug("setting to %u kHz\n", policy->max); + __cpufreq_driver_target(policy, policy->max, CPUFREQ_RELATION_H); } -#ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE_MODULE -static -#endif -struct cpufreq_governor cpufreq_gov_performance = { +static struct cpufreq_governor cpufreq_gov_performance = { .name = "performance", - .governor = cpufreq_governor_performance, .owner = THIS_MODULE, + .flags = CPUFREQ_GOV_STRICT_TARGET, + .limits = cpufreq_gov_performance_limits, }; -static int __init cpufreq_gov_performance_init(void) +#ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE +struct cpufreq_governor *cpufreq_default_governor(void) { - return cpufreq_register_governor(&cpufreq_gov_performance); + return &cpufreq_gov_performance; } - -static void __exit cpufreq_gov_performance_exit(void) +#endif +#ifndef CONFIG_CPU_FREQ_GOV_PERFORMANCE_MODULE +struct cpufreq_governor *cpufreq_fallback_governor(void) { - cpufreq_unregister_governor(&cpufreq_gov_performance); + return &cpufreq_gov_performance; } +#endif MODULE_AUTHOR("Dominik Brodowski <linux@brodo.de>"); MODULE_DESCRIPTION("CPUfreq policy governor 'performance'"); MODULE_LICENSE("GPL"); -fs_initcall(cpufreq_gov_performance_init); -module_exit(cpufreq_gov_performance_exit); +cpufreq_governor_init(cpufreq_gov_performance); +cpufreq_governor_exit(cpufreq_gov_performance); diff --git a/drivers/cpufreq/cpufreq_powersave.c b/drivers/cpufreq/cpufreq_powersave.c index 32109a14f5dc..8d830d860e91 100644 --- a/drivers/cpufreq/cpufreq_powersave.c +++ b/drivers/cpufreq/cpufreq_powersave.c @@ -1,65 +1,39 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * linux/drivers/cpufreq/cpufreq_powersave.c * * Copyright (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de> - * - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -#include <linux/kernel.h> -#include <linux/module.h> #include <linux/cpufreq.h> #include <linux/init.h> +#include <linux/module.h> -static int cpufreq_governor_powersave(struct cpufreq_policy *policy, - unsigned int event) +static void cpufreq_gov_powersave_limits(struct cpufreq_policy *policy) { - switch (event) { - case CPUFREQ_GOV_START: - case CPUFREQ_GOV_LIMITS: - pr_debug("setting to %u kHz because of event %u\n", - policy->min, event); - __cpufreq_driver_target(policy, policy->min, - CPUFREQ_RELATION_L); - break; - default: - break; - } - return 0; + pr_debug("setting to %u kHz\n", policy->min); + __cpufreq_driver_target(policy, policy->min, CPUFREQ_RELATION_L); } -#ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_POWERSAVE -static -#endif -struct cpufreq_governor cpufreq_gov_powersave = { +static struct cpufreq_governor cpufreq_gov_powersave = { .name = "powersave", - .governor = cpufreq_governor_powersave, + .limits = cpufreq_gov_powersave_limits, .owner = THIS_MODULE, + .flags = CPUFREQ_GOV_STRICT_TARGET, }; -static int __init cpufreq_gov_powersave_init(void) -{ - return cpufreq_register_governor(&cpufreq_gov_powersave); -} - -static void __exit cpufreq_gov_powersave_exit(void) -{ - cpufreq_unregister_governor(&cpufreq_gov_powersave); -} - MODULE_AUTHOR("Dominik Brodowski <linux@brodo.de>"); MODULE_DESCRIPTION("CPUfreq policy governor 'powersave'"); MODULE_LICENSE("GPL"); #ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_POWERSAVE -fs_initcall(cpufreq_gov_powersave_init); -#else -module_init(cpufreq_gov_powersave_init); +struct cpufreq_governor *cpufreq_default_governor(void) +{ + return &cpufreq_gov_powersave; +} #endif -module_exit(cpufreq_gov_powersave_exit); + +cpufreq_governor_init(cpufreq_gov_powersave); +cpufreq_governor_exit(cpufreq_gov_powersave); diff --git a/drivers/cpufreq/cpufreq_stats.c b/drivers/cpufreq/cpufreq_stats.c index cd9e81713a71..40a9ff18da06 100644 --- a/drivers/cpufreq/cpufreq_stats.c +++ b/drivers/cpufreq/cpufreq_stats.c @@ -1,31 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * drivers/cpufreq/cpufreq_stats.c * * Copyright (C) 2003-2004 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>. * (C) 2004 Zou Nan hai <nanhai.zou@intel.com>. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ -#include <linux/kernel.h> -#include <linux/slab.h> #include <linux/cpu.h> -#include <linux/sysfs.h> #include <linux/cpufreq.h> #include <linux/module.h> -#include <linux/jiffies.h> -#include <linux/percpu.h> -#include <linux/kobject.h> -#include <linux/spinlock.h> -#include <linux/notifier.h> -#include <asm/cputime.h> - -static spinlock_t cpufreq_stats_lock; +#include <linux/sched/clock.h> +#include <linux/slab.h> struct cpufreq_stats { - unsigned int cpu; unsigned int total_trans; unsigned long long last_time; unsigned int max_state; @@ -33,400 +20,270 @@ struct cpufreq_stats { unsigned int last_index; u64 *time_in_state; unsigned int *freq_table; -#ifdef CONFIG_CPU_FREQ_STAT_DETAILS unsigned int *trans_table; -#endif + + /* Deferred reset */ + unsigned int reset_pending; + unsigned long long reset_time; }; -static DEFINE_PER_CPU(struct cpufreq_stats *, cpufreq_stats_table); +static void cpufreq_stats_update(struct cpufreq_stats *stats, + unsigned long long time) +{ + unsigned long long cur_time = local_clock(); -struct cpufreq_stats_attribute { - struct attribute attr; - ssize_t(*show) (struct cpufreq_stats *, char *); -}; + stats->time_in_state[stats->last_index] += cur_time - time; + stats->last_time = cur_time; +} -static int cpufreq_stats_update(unsigned int cpu) +static void cpufreq_stats_reset_table(struct cpufreq_stats *stats) { - struct cpufreq_stats *stat; - unsigned long long cur_time; - - cur_time = get_jiffies_64(); - spin_lock(&cpufreq_stats_lock); - stat = per_cpu(cpufreq_stats_table, cpu); - if (stat->time_in_state) - stat->time_in_state[stat->last_index] += - cur_time - stat->last_time; - stat->last_time = cur_time; - spin_unlock(&cpufreq_stats_lock); - return 0; + unsigned int count = stats->max_state; + + memset(stats->time_in_state, 0, count * sizeof(u64)); + memset(stats->trans_table, 0, count * count * sizeof(int)); + stats->last_time = local_clock(); + stats->total_trans = 0; + + /* Adjust for the time elapsed since reset was requested */ + WRITE_ONCE(stats->reset_pending, 0); + /* + * Prevent the reset_time read from being reordered before the + * reset_pending accesses in cpufreq_stats_record_transition(). + */ + smp_rmb(); + cpufreq_stats_update(stats, READ_ONCE(stats->reset_time)); } static ssize_t show_total_trans(struct cpufreq_policy *policy, char *buf) { - struct cpufreq_stats *stat = per_cpu(cpufreq_stats_table, policy->cpu); - if (!stat) - return 0; - return sprintf(buf, "%d\n", - per_cpu(cpufreq_stats_table, stat->cpu)->total_trans); + struct cpufreq_stats *stats = policy->stats; + + if (READ_ONCE(stats->reset_pending)) + return sprintf(buf, "%d\n", 0); + else + return sprintf(buf, "%u\n", stats->total_trans); } +cpufreq_freq_attr_ro(total_trans); static ssize_t show_time_in_state(struct cpufreq_policy *policy, char *buf) { + struct cpufreq_stats *stats = policy->stats; + bool pending = READ_ONCE(stats->reset_pending); + unsigned long long time; ssize_t len = 0; int i; - struct cpufreq_stats *stat = per_cpu(cpufreq_stats_table, policy->cpu); - if (!stat) - return 0; - cpufreq_stats_update(stat->cpu); - for (i = 0; i < stat->state_num; i++) { - len += sprintf(buf + len, "%u %llu\n", stat->freq_table[i], - (unsigned long long) - cputime64_to_clock_t(stat->time_in_state[i])); + + for (i = 0; i < stats->state_num; i++) { + if (pending) { + if (i == stats->last_index) { + /* + * Prevent the reset_time read from occurring + * before the reset_pending read above. + */ + smp_rmb(); + time = local_clock() - READ_ONCE(stats->reset_time); + } else { + time = 0; + } + } else { + time = stats->time_in_state[i]; + if (i == stats->last_index) + time += local_clock() - stats->last_time; + } + + len += sprintf(buf + len, "%u %llu\n", stats->freq_table[i], + nsec_to_clock_t(time)); } return len; } +cpufreq_freq_attr_ro(time_in_state); + +/* We don't care what is written to the attribute */ +static ssize_t store_reset(struct cpufreq_policy *policy, const char *buf, + size_t count) +{ + struct cpufreq_stats *stats = policy->stats; + + /* + * Defer resetting of stats to cpufreq_stats_record_transition() to + * avoid races. + */ + WRITE_ONCE(stats->reset_time, local_clock()); + /* + * The memory barrier below is to prevent the readers of reset_time from + * seeing a stale or partially updated value. + */ + smp_wmb(); + WRITE_ONCE(stats->reset_pending, 1); + + return count; +} +cpufreq_freq_attr_wo(reset); -#ifdef CONFIG_CPU_FREQ_STAT_DETAILS static ssize_t show_trans_table(struct cpufreq_policy *policy, char *buf) { + struct cpufreq_stats *stats = policy->stats; + bool pending = READ_ONCE(stats->reset_pending); ssize_t len = 0; - int i, j; - - struct cpufreq_stats *stat = per_cpu(cpufreq_stats_table, policy->cpu); - if (!stat) - return 0; - cpufreq_stats_update(stat->cpu); - len += snprintf(buf + len, PAGE_SIZE - len, " From : To\n"); - len += snprintf(buf + len, PAGE_SIZE - len, " : "); - for (i = 0; i < stat->state_num; i++) { - if (len >= PAGE_SIZE) + int i, j, count; + + len += sysfs_emit_at(buf, len, " From : To\n"); + len += sysfs_emit_at(buf, len, " : "); + for (i = 0; i < stats->state_num; i++) { + if (len >= PAGE_SIZE - 1) break; - len += snprintf(buf + len, PAGE_SIZE - len, "%9u ", - stat->freq_table[i]); + len += sysfs_emit_at(buf, len, "%9u ", stats->freq_table[i]); } - if (len >= PAGE_SIZE) - return PAGE_SIZE; + if (len >= PAGE_SIZE - 1) + return PAGE_SIZE - 1; - len += snprintf(buf + len, PAGE_SIZE - len, "\n"); + len += sysfs_emit_at(buf, len, "\n"); - for (i = 0; i < stat->state_num; i++) { - if (len >= PAGE_SIZE) + for (i = 0; i < stats->state_num; i++) { + if (len >= PAGE_SIZE - 1) break; - len += snprintf(buf + len, PAGE_SIZE - len, "%9u: ", - stat->freq_table[i]); + len += sysfs_emit_at(buf, len, "%9u: ", stats->freq_table[i]); - for (j = 0; j < stat->state_num; j++) { - if (len >= PAGE_SIZE) + for (j = 0; j < stats->state_num; j++) { + if (len >= PAGE_SIZE - 1) break; - len += snprintf(buf + len, PAGE_SIZE - len, "%9u ", - stat->trans_table[i*stat->max_state+j]); + + if (pending) + count = 0; + else + count = stats->trans_table[i * stats->max_state + j]; + + len += sysfs_emit_at(buf, len, "%9u ", count); } - if (len >= PAGE_SIZE) + if (len >= PAGE_SIZE - 1) break; - len += snprintf(buf + len, PAGE_SIZE - len, "\n"); + len += sysfs_emit_at(buf, len, "\n"); + } + + if (len >= PAGE_SIZE - 1) { + pr_warn_once("cpufreq transition table exceeds PAGE_SIZE. Disabling\n"); + return -EFBIG; } - if (len >= PAGE_SIZE) - return PAGE_SIZE; return len; } cpufreq_freq_attr_ro(trans_table); -#endif - -cpufreq_freq_attr_ro(total_trans); -cpufreq_freq_attr_ro(time_in_state); static struct attribute *default_attrs[] = { &total_trans.attr, &time_in_state.attr, -#ifdef CONFIG_CPU_FREQ_STAT_DETAILS + &reset.attr, &trans_table.attr, -#endif NULL }; -static struct attribute_group stats_attr_group = { +static const struct attribute_group stats_attr_group = { .attrs = default_attrs, .name = "stats" }; -static int freq_table_get_index(struct cpufreq_stats *stat, unsigned int freq) +static int freq_table_get_index(struct cpufreq_stats *stats, unsigned int freq) { int index; - for (index = 0; index < stat->max_state; index++) - if (stat->freq_table[index] == freq) + for (index = 0; index < stats->max_state; index++) + if (stats->freq_table[index] == freq) return index; return -1; } -/* should be called late in the CPU removal sequence so that the stats - * memory is still available in case someone tries to use it. - */ -static void cpufreq_stats_free_table(unsigned int cpu) +void cpufreq_stats_free_table(struct cpufreq_policy *policy) { - struct cpufreq_stats *stat = per_cpu(cpufreq_stats_table, cpu); + struct cpufreq_stats *stats = policy->stats; - if (stat) { - pr_debug("%s: Free stat table\n", __func__); - kfree(stat->time_in_state); - kfree(stat); - per_cpu(cpufreq_stats_table, cpu) = NULL; - } -} - -/* must be called early in the CPU removal sequence (before - * cpufreq_remove_dev) so that policy is still valid. - */ -static void cpufreq_stats_free_sysfs(unsigned int cpu) -{ - struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); - - if (!policy) + /* Already freed */ + if (!stats) return; - if (!cpufreq_frequency_get_table(cpu)) - goto put_ref; - - if (!policy_is_shared(policy)) { - pr_debug("%s: Free sysfs stat\n", __func__); - sysfs_remove_group(&policy->kobj, &stats_attr_group); - } + pr_debug("%s: Free stats table\n", __func__); -put_ref: - cpufreq_cpu_put(policy); + sysfs_remove_group(&policy->kobj, &stats_attr_group); + kfree(stats->time_in_state); + kfree(stats); + policy->stats = NULL; } -static int cpufreq_stats_create_table(struct cpufreq_policy *policy, - struct cpufreq_frequency_table *table) +void cpufreq_stats_create_table(struct cpufreq_policy *policy) { - unsigned int i, j, count = 0, ret = 0; - struct cpufreq_stats *stat; - struct cpufreq_policy *data; + unsigned int i = 0, count; + struct cpufreq_stats *stats; unsigned int alloc_size; - unsigned int cpu = policy->cpu; - if (per_cpu(cpufreq_stats_table, cpu)) - return -EBUSY; - stat = kzalloc(sizeof(struct cpufreq_stats), GFP_KERNEL); - if ((stat) == NULL) - return -ENOMEM; - - data = cpufreq_cpu_get(cpu); - if (data == NULL) { - ret = -EINVAL; - goto error_get_fail; - } + struct cpufreq_frequency_table *pos; - ret = sysfs_create_group(&data->kobj, &stats_attr_group); - if (ret) - goto error_out; + count = cpufreq_table_count_valid_entries(policy); + if (!count) + return; - stat->cpu = cpu; - per_cpu(cpufreq_stats_table, cpu) = stat; + /* stats already initialized */ + if (policy->stats) + return; - for (i = 0; table[i].frequency != CPUFREQ_TABLE_END; i++) { - unsigned int freq = table[i].frequency; - if (freq == CPUFREQ_ENTRY_INVALID) - continue; - count++; - } + stats = kzalloc(sizeof(*stats), GFP_KERNEL); + if (!stats) + return; alloc_size = count * sizeof(int) + count * sizeof(u64); -#ifdef CONFIG_CPU_FREQ_STAT_DETAILS alloc_size += count * count * sizeof(int); -#endif - stat->max_state = count; - stat->time_in_state = kzalloc(alloc_size, GFP_KERNEL); - if (!stat->time_in_state) { - ret = -ENOMEM; - goto error_out; - } - stat->freq_table = (unsigned int *)(stat->time_in_state + count); - -#ifdef CONFIG_CPU_FREQ_STAT_DETAILS - stat->trans_table = stat->freq_table + count; -#endif - j = 0; - for (i = 0; table[i].frequency != CPUFREQ_TABLE_END; i++) { - unsigned int freq = table[i].frequency; - if (freq == CPUFREQ_ENTRY_INVALID) - continue; - if (freq_table_get_index(stat, freq) == -1) - stat->freq_table[j++] = freq; - } - stat->state_num = j; - spin_lock(&cpufreq_stats_lock); - stat->last_time = get_jiffies_64(); - stat->last_index = freq_table_get_index(stat, policy->cur); - spin_unlock(&cpufreq_stats_lock); - cpufreq_cpu_put(data); - return 0; -error_out: - cpufreq_cpu_put(data); -error_get_fail: - kfree(stat); - per_cpu(cpufreq_stats_table, cpu) = NULL; - return ret; -} -static void cpufreq_stats_update_policy_cpu(struct cpufreq_policy *policy) -{ - struct cpufreq_stats *stat = per_cpu(cpufreq_stats_table, - policy->last_cpu); - - pr_debug("Updating stats_table for new_cpu %u from last_cpu %u\n", - policy->cpu, policy->last_cpu); - per_cpu(cpufreq_stats_table, policy->cpu) = per_cpu(cpufreq_stats_table, - policy->last_cpu); - per_cpu(cpufreq_stats_table, policy->last_cpu) = NULL; - stat->cpu = policy->cpu; -} - -static int cpufreq_stat_notifier_policy(struct notifier_block *nb, - unsigned long val, void *data) -{ - int ret; - struct cpufreq_policy *policy = data; - struct cpufreq_frequency_table *table; - unsigned int cpu = policy->cpu; - - if (val == CPUFREQ_UPDATE_POLICY_CPU) { - cpufreq_stats_update_policy_cpu(policy); - return 0; - } - - if (val != CPUFREQ_NOTIFY) - return 0; - table = cpufreq_frequency_get_table(cpu); - if (!table) - return 0; - ret = cpufreq_stats_create_table(policy, table); - if (ret) - return ret; - return 0; -} - -static int cpufreq_stat_notifier_trans(struct notifier_block *nb, - unsigned long val, void *data) -{ - struct cpufreq_freqs *freq = data; - struct cpufreq_stats *stat; - int old_index, new_index; + /* Allocate memory for time_in_state/freq_table/trans_table in one go */ + stats->time_in_state = kzalloc(alloc_size, GFP_KERNEL); + if (!stats->time_in_state) + goto free_stat; - if (val != CPUFREQ_POSTCHANGE) - return 0; + stats->freq_table = (unsigned int *)(stats->time_in_state + count); - stat = per_cpu(cpufreq_stats_table, freq->cpu); - if (!stat) - return 0; + stats->trans_table = stats->freq_table + count; - old_index = stat->last_index; - new_index = freq_table_get_index(stat, freq->new); + stats->max_state = count; - /* We can't do stat->time_in_state[-1]= .. */ - if (old_index == -1 || new_index == -1) - return 0; + /* Find valid-unique entries */ + cpufreq_for_each_valid_entry(pos, policy->freq_table) + if (policy->freq_table_sorted != CPUFREQ_TABLE_UNSORTED || + freq_table_get_index(stats, pos->frequency) == -1) + stats->freq_table[i++] = pos->frequency; - cpufreq_stats_update(freq->cpu); + stats->state_num = i; + stats->last_time = local_clock(); + stats->last_index = freq_table_get_index(stats, policy->cur); - if (old_index == new_index) - return 0; + policy->stats = stats; + if (!sysfs_create_group(&policy->kobj, &stats_attr_group)) + return; - spin_lock(&cpufreq_stats_lock); - stat->last_index = new_index; -#ifdef CONFIG_CPU_FREQ_STAT_DETAILS - stat->trans_table[old_index * stat->max_state + new_index]++; -#endif - stat->total_trans++; - spin_unlock(&cpufreq_stats_lock); - return 0; + /* We failed, release resources */ + policy->stats = NULL; + kfree(stats->time_in_state); +free_stat: + kfree(stats); } -static int __cpuinit cpufreq_stat_cpu_callback(struct notifier_block *nfb, - unsigned long action, - void *hcpu) +void cpufreq_stats_record_transition(struct cpufreq_policy *policy, + unsigned int new_freq) { - unsigned int cpu = (unsigned long)hcpu; - - switch (action) { - case CPU_ONLINE: - case CPU_ONLINE_FROZEN: - cpufreq_update_policy(cpu); - break; - case CPU_DOWN_PREPARE: - cpufreq_stats_free_sysfs(cpu); - break; - case CPU_DEAD: - cpufreq_stats_free_table(cpu); - break; - case CPU_UP_CANCELED_FROZEN: - cpufreq_stats_free_sysfs(cpu); - cpufreq_stats_free_table(cpu); - break; - } - return NOTIFY_OK; -} + struct cpufreq_stats *stats = policy->stats; + int old_index, new_index; -/* priority=1 so this will get called before cpufreq_remove_dev */ -static struct notifier_block cpufreq_stat_cpu_notifier __refdata = { - .notifier_call = cpufreq_stat_cpu_callback, - .priority = 1, -}; + if (unlikely(!stats)) + return; -static struct notifier_block notifier_policy_block = { - .notifier_call = cpufreq_stat_notifier_policy -}; + if (unlikely(READ_ONCE(stats->reset_pending))) + cpufreq_stats_reset_table(stats); -static struct notifier_block notifier_trans_block = { - .notifier_call = cpufreq_stat_notifier_trans -}; + old_index = stats->last_index; + new_index = freq_table_get_index(stats, new_freq); -static int __init cpufreq_stats_init(void) -{ - int ret; - unsigned int cpu; - - spin_lock_init(&cpufreq_stats_lock); - ret = cpufreq_register_notifier(¬ifier_policy_block, - CPUFREQ_POLICY_NOTIFIER); - if (ret) - return ret; - - register_hotcpu_notifier(&cpufreq_stat_cpu_notifier); - for_each_online_cpu(cpu) - cpufreq_update_policy(cpu); - - ret = cpufreq_register_notifier(¬ifier_trans_block, - CPUFREQ_TRANSITION_NOTIFIER); - if (ret) { - cpufreq_unregister_notifier(¬ifier_policy_block, - CPUFREQ_POLICY_NOTIFIER); - unregister_hotcpu_notifier(&cpufreq_stat_cpu_notifier); - for_each_online_cpu(cpu) - cpufreq_stats_free_table(cpu); - return ret; - } - - return 0; -} -static void __exit cpufreq_stats_exit(void) -{ - unsigned int cpu; - - cpufreq_unregister_notifier(¬ifier_policy_block, - CPUFREQ_POLICY_NOTIFIER); - cpufreq_unregister_notifier(¬ifier_trans_block, - CPUFREQ_TRANSITION_NOTIFIER); - unregister_hotcpu_notifier(&cpufreq_stat_cpu_notifier); - for_each_online_cpu(cpu) { - cpufreq_stats_free_table(cpu); - cpufreq_stats_free_sysfs(cpu); - } -} + /* We can't do stats->time_in_state[-1]= .. */ + if (unlikely(old_index == -1 || new_index == -1 || old_index == new_index)) + return; -MODULE_AUTHOR("Zou Nan hai <nanhai.zou@intel.com>"); -MODULE_DESCRIPTION("'cpufreq_stats' - A driver to export cpufreq stats " - "through sysfs filesystem"); -MODULE_LICENSE("GPL"); + cpufreq_stats_update(stats, stats->last_time); -module_init(cpufreq_stats_init); -module_exit(cpufreq_stats_exit); + stats->last_index = new_index; + stats->trans_table[old_index * stats->max_state + new_index]++; + stats->total_trans++; +} diff --git a/drivers/cpufreq/cpufreq_userspace.c b/drivers/cpufreq/cpufreq_userspace.c index 03078090b5f7..77d62152cd38 100644 --- a/drivers/cpufreq/cpufreq_userspace.c +++ b/drivers/cpufreq/cpufreq_userspace.c @@ -1,14 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * linux/drivers/cpufreq/cpufreq_userspace.c * * Copyright (C) 2001 Russell King * (C) 2002 - 2004 Dominik Brodowski <linux@brodo.de> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt @@ -17,9 +13,13 @@ #include <linux/init.h> #include <linux/module.h> #include <linux/mutex.h> +#include <linux/slab.h> -static DEFINE_PER_CPU(unsigned int, cpu_is_managed); -static DEFINE_MUTEX(userspace_mutex); +struct userspace_policy { + unsigned int is_managed; + unsigned int setspeed; + struct mutex mutex; +}; /** * cpufreq_set - set the CPU frequency @@ -31,27 +31,19 @@ static DEFINE_MUTEX(userspace_mutex); static int cpufreq_set(struct cpufreq_policy *policy, unsigned int freq) { int ret = -EINVAL; + struct userspace_policy *userspace = policy->governor_data; pr_debug("cpufreq_set for cpu %u, freq %u kHz\n", policy->cpu, freq); - mutex_lock(&userspace_mutex); - if (!per_cpu(cpu_is_managed, policy->cpu)) + mutex_lock(&userspace->mutex); + if (!userspace->is_managed) goto err; - /* - * We're safe from concurrent calls to ->target() here - * as we hold the userspace_mutex lock. If we were calling - * cpufreq_driver_target, a deadlock situation might occur: - * A: cpufreq_set (lock userspace_mutex) -> - * cpufreq_driver_target(lock policy->lock) - * B: cpufreq_set_policy(lock policy->lock) -> - * __cpufreq_governor -> - * cpufreq_governor_userspace (lock userspace_mutex) - */ - ret = __cpufreq_driver_target(policy, freq, CPUFREQ_RELATION_L); + userspace->setspeed = freq; + ret = __cpufreq_driver_target(policy, freq, CPUFREQ_RELATION_L); err: - mutex_unlock(&userspace_mutex); + mutex_unlock(&userspace->mutex); return ret; } @@ -60,75 +52,102 @@ static ssize_t show_speed(struct cpufreq_policy *policy, char *buf) return sprintf(buf, "%u\n", policy->cur); } -static int cpufreq_governor_userspace(struct cpufreq_policy *policy, - unsigned int event) +static int cpufreq_userspace_policy_init(struct cpufreq_policy *policy) { - unsigned int cpu = policy->cpu; - int rc = 0; - - switch (event) { - case CPUFREQ_GOV_START: - BUG_ON(!policy->cur); - pr_debug("started managing cpu %u\n", cpu); - - mutex_lock(&userspace_mutex); - per_cpu(cpu_is_managed, cpu) = 1; - mutex_unlock(&userspace_mutex); - break; - case CPUFREQ_GOV_STOP: - pr_debug("managing cpu %u stopped\n", cpu); - - mutex_lock(&userspace_mutex); - per_cpu(cpu_is_managed, cpu) = 0; - mutex_unlock(&userspace_mutex); - break; - case CPUFREQ_GOV_LIMITS: - mutex_lock(&userspace_mutex); - pr_debug("limit event for cpu %u: %u - %u kHz, currently %u kHz\n", - cpu, policy->min, policy->max, - policy->cur); - - if (policy->max < policy->cur) - __cpufreq_driver_target(policy, policy->max, - CPUFREQ_RELATION_H); - else if (policy->min > policy->cur) - __cpufreq_driver_target(policy, policy->min, - CPUFREQ_RELATION_L); - mutex_unlock(&userspace_mutex); - break; - } - return rc; + struct userspace_policy *userspace; + + userspace = kzalloc(sizeof(*userspace), GFP_KERNEL); + if (!userspace) + return -ENOMEM; + + mutex_init(&userspace->mutex); + + policy->governor_data = userspace; + return 0; } -#ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_USERSPACE -static -#endif -struct cpufreq_governor cpufreq_gov_userspace = { - .name = "userspace", - .governor = cpufreq_governor_userspace, - .store_setspeed = cpufreq_set, - .show_setspeed = show_speed, - .owner = THIS_MODULE, -}; +/* + * Any routine that writes to the policy struct will hold the "rwsem" of + * policy struct that means it is free to free "governor_data" here. + */ +static void cpufreq_userspace_policy_exit(struct cpufreq_policy *policy) +{ + kfree(policy->governor_data); + policy->governor_data = NULL; +} + +static int cpufreq_userspace_policy_start(struct cpufreq_policy *policy) +{ + struct userspace_policy *userspace = policy->governor_data; + + BUG_ON(!policy->cur); + pr_debug("started managing cpu %u\n", policy->cpu); -static int __init cpufreq_gov_userspace_init(void) + mutex_lock(&userspace->mutex); + userspace->is_managed = 1; + userspace->setspeed = policy->cur; + mutex_unlock(&userspace->mutex); + return 0; +} + +static void cpufreq_userspace_policy_stop(struct cpufreq_policy *policy) { - return cpufreq_register_governor(&cpufreq_gov_userspace); + struct userspace_policy *userspace = policy->governor_data; + + pr_debug("managing cpu %u stopped\n", policy->cpu); + + mutex_lock(&userspace->mutex); + userspace->is_managed = 0; + userspace->setspeed = 0; + mutex_unlock(&userspace->mutex); } -static void __exit cpufreq_gov_userspace_exit(void) +static void cpufreq_userspace_policy_limits(struct cpufreq_policy *policy) { - cpufreq_unregister_governor(&cpufreq_gov_userspace); + struct userspace_policy *userspace = policy->governor_data; + + mutex_lock(&userspace->mutex); + + pr_debug("limit event for cpu %u: %u - %u kHz, currently %u kHz, last set to %u kHz\n", + policy->cpu, policy->min, policy->max, policy->cur, userspace->setspeed); + + if (policy->max < userspace->setspeed) + __cpufreq_driver_target(policy, policy->max, + CPUFREQ_RELATION_H); + else if (policy->min > userspace->setspeed) + __cpufreq_driver_target(policy, policy->min, + CPUFREQ_RELATION_L); + else + __cpufreq_driver_target(policy, userspace->setspeed, + CPUFREQ_RELATION_L); + + mutex_unlock(&userspace->mutex); } +static struct cpufreq_governor cpufreq_gov_userspace = { + .name = "userspace", + .init = cpufreq_userspace_policy_init, + .exit = cpufreq_userspace_policy_exit, + .start = cpufreq_userspace_policy_start, + .stop = cpufreq_userspace_policy_stop, + .limits = cpufreq_userspace_policy_limits, + .store_setspeed = cpufreq_set, + .show_setspeed = show_speed, + .owner = THIS_MODULE, + .flags = CPUFREQ_GOV_STRICT_TARGET, +}; + MODULE_AUTHOR("Dominik Brodowski <linux@brodo.de>, " "Russell King <rmk@arm.linux.org.uk>"); MODULE_DESCRIPTION("CPUfreq policy governor 'userspace'"); MODULE_LICENSE("GPL"); #ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_USERSPACE -fs_initcall(cpufreq_gov_userspace_init); -#else -module_init(cpufreq_gov_userspace_init); +struct cpufreq_governor *cpufreq_default_governor(void) +{ + return &cpufreq_gov_userspace; +} #endif -module_exit(cpufreq_gov_userspace_exit); + +cpufreq_governor_init(cpufreq_gov_userspace); +cpufreq_governor_exit(cpufreq_gov_userspace); diff --git a/drivers/cpufreq/cris-artpec3-cpufreq.c b/drivers/cpufreq/cris-artpec3-cpufreq.c deleted file mode 100644 index ee142c490575..000000000000 --- a/drivers/cpufreq/cris-artpec3-cpufreq.c +++ /dev/null @@ -1,146 +0,0 @@ -#include <linux/init.h> -#include <linux/module.h> -#include <linux/cpufreq.h> -#include <hwregs/reg_map.h> -#include <hwregs/reg_rdwr.h> -#include <hwregs/clkgen_defs.h> -#include <hwregs/ddr2_defs.h> - -static int -cris_sdram_freq_notifier(struct notifier_block *nb, unsigned long val, - void *data); - -static struct notifier_block cris_sdram_freq_notifier_block = { - .notifier_call = cris_sdram_freq_notifier -}; - -static struct cpufreq_frequency_table cris_freq_table[] = { - {0x01, 6000}, - {0x02, 200000}, - {0, CPUFREQ_TABLE_END}, -}; - -static unsigned int cris_freq_get_cpu_frequency(unsigned int cpu) -{ - reg_clkgen_rw_clk_ctrl clk_ctrl; - clk_ctrl = REG_RD(clkgen, regi_clkgen, rw_clk_ctrl); - return clk_ctrl.pll ? 200000 : 6000; -} - -static void cris_freq_set_cpu_state(struct cpufreq_policy *policy, - unsigned int state) -{ - struct cpufreq_freqs freqs; - reg_clkgen_rw_clk_ctrl clk_ctrl; - clk_ctrl = REG_RD(clkgen, regi_clkgen, rw_clk_ctrl); - - freqs.old = cris_freq_get_cpu_frequency(policy->cpu); - freqs.new = cris_freq_table[state].frequency; - - cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE); - - local_irq_disable(); - - /* Even though we may be SMP they will share the same clock - * so all settings are made on CPU0. */ - if (cris_freq_table[state].frequency == 200000) - clk_ctrl.pll = 1; - else - clk_ctrl.pll = 0; - REG_WR(clkgen, regi_clkgen, rw_clk_ctrl, clk_ctrl); - - local_irq_enable(); - - cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); -}; - -static int cris_freq_verify(struct cpufreq_policy *policy) -{ - return cpufreq_frequency_table_verify(policy, &cris_freq_table[0]); -} - -static int cris_freq_target(struct cpufreq_policy *policy, - unsigned int target_freq, - unsigned int relation) -{ - unsigned int newstate = 0; - - if (cpufreq_frequency_table_target(policy, cris_freq_table, - target_freq, relation, &newstate)) - return -EINVAL; - - cris_freq_set_cpu_state(policy, newstate); - - return 0; -} - -static int cris_freq_cpu_init(struct cpufreq_policy *policy) -{ - int result; - - /* cpuinfo and default policy values */ - policy->cpuinfo.transition_latency = 1000000; /* 1ms */ - policy->cur = cris_freq_get_cpu_frequency(0); - - result = cpufreq_frequency_table_cpuinfo(policy, cris_freq_table); - if (result) - return (result); - - cpufreq_frequency_table_get_attr(cris_freq_table, policy->cpu); - - return 0; -} - - -static int cris_freq_cpu_exit(struct cpufreq_policy *policy) -{ - cpufreq_frequency_table_put_attr(policy->cpu); - return 0; -} - - -static struct freq_attr *cris_freq_attr[] = { - &cpufreq_freq_attr_scaling_available_freqs, - NULL, -}; - -static struct cpufreq_driver cris_freq_driver = { - .get = cris_freq_get_cpu_frequency, - .verify = cris_freq_verify, - .target = cris_freq_target, - .init = cris_freq_cpu_init, - .exit = cris_freq_cpu_exit, - .name = "cris_freq", - .owner = THIS_MODULE, - .attr = cris_freq_attr, -}; - -static int __init cris_freq_init(void) -{ - int ret; - ret = cpufreq_register_driver(&cris_freq_driver); - cpufreq_register_notifier(&cris_sdram_freq_notifier_block, - CPUFREQ_TRANSITION_NOTIFIER); - return ret; -} - -static int -cris_sdram_freq_notifier(struct notifier_block *nb, unsigned long val, - void *data) -{ - int i; - struct cpufreq_freqs *freqs = data; - if (val == CPUFREQ_PRECHANGE) { - reg_ddr2_rw_cfg cfg = - REG_RD(ddr2, regi_ddr2_ctrl, rw_cfg); - cfg.ref_interval = (freqs->new == 200000 ? 1560 : 46); - - if (freqs->new == 200000) - for (i = 0; i < 50000; i++); - REG_WR(bif_core, regi_bif_core, rw_sdram_timing, timing); - } - return 0; -} - - -module_init(cris_freq_init); diff --git a/drivers/cpufreq/cris-etraxfs-cpufreq.c b/drivers/cpufreq/cris-etraxfs-cpufreq.c deleted file mode 100644 index 12952235d5db..000000000000 --- a/drivers/cpufreq/cris-etraxfs-cpufreq.c +++ /dev/null @@ -1,142 +0,0 @@ -#include <linux/init.h> -#include <linux/module.h> -#include <linux/cpufreq.h> -#include <hwregs/reg_map.h> -#include <arch/hwregs/reg_rdwr.h> -#include <arch/hwregs/config_defs.h> -#include <arch/hwregs/bif_core_defs.h> - -static int -cris_sdram_freq_notifier(struct notifier_block *nb, unsigned long val, - void *data); - -static struct notifier_block cris_sdram_freq_notifier_block = { - .notifier_call = cris_sdram_freq_notifier -}; - -static struct cpufreq_frequency_table cris_freq_table[] = { - {0x01, 6000}, - {0x02, 200000}, - {0, CPUFREQ_TABLE_END}, -}; - -static unsigned int cris_freq_get_cpu_frequency(unsigned int cpu) -{ - reg_config_rw_clk_ctrl clk_ctrl; - clk_ctrl = REG_RD(config, regi_config, rw_clk_ctrl); - return clk_ctrl.pll ? 200000 : 6000; -} - -static void cris_freq_set_cpu_state(struct cpufreq_policy *policy, - unsigned int state) -{ - struct cpufreq_freqs freqs; - reg_config_rw_clk_ctrl clk_ctrl; - clk_ctrl = REG_RD(config, regi_config, rw_clk_ctrl); - - freqs.old = cris_freq_get_cpu_frequency(policy->cpu); - freqs.new = cris_freq_table[state].frequency; - - cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE); - - local_irq_disable(); - - /* Even though we may be SMP they will share the same clock - * so all settings are made on CPU0. */ - if (cris_freq_table[state].frequency == 200000) - clk_ctrl.pll = 1; - else - clk_ctrl.pll = 0; - REG_WR(config, regi_config, rw_clk_ctrl, clk_ctrl); - - local_irq_enable(); - - cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); -}; - -static int cris_freq_verify(struct cpufreq_policy *policy) -{ - return cpufreq_frequency_table_verify(policy, &cris_freq_table[0]); -} - -static int cris_freq_target(struct cpufreq_policy *policy, - unsigned int target_freq, unsigned int relation) -{ - unsigned int newstate = 0; - - if (cpufreq_frequency_table_target - (policy, cris_freq_table, target_freq, relation, &newstate)) - return -EINVAL; - - cris_freq_set_cpu_state(policy, newstate); - - return 0; -} - -static int cris_freq_cpu_init(struct cpufreq_policy *policy) -{ - int result; - - /* cpuinfo and default policy values */ - policy->cpuinfo.transition_latency = 1000000; /* 1ms */ - policy->cur = cris_freq_get_cpu_frequency(0); - - result = cpufreq_frequency_table_cpuinfo(policy, cris_freq_table); - if (result) - return (result); - - cpufreq_frequency_table_get_attr(cris_freq_table, policy->cpu); - - return 0; -} - -static int cris_freq_cpu_exit(struct cpufreq_policy *policy) -{ - cpufreq_frequency_table_put_attr(policy->cpu); - return 0; -} - -static struct freq_attr *cris_freq_attr[] = { - &cpufreq_freq_attr_scaling_available_freqs, - NULL, -}; - -static struct cpufreq_driver cris_freq_driver = { - .get = cris_freq_get_cpu_frequency, - .verify = cris_freq_verify, - .target = cris_freq_target, - .init = cris_freq_cpu_init, - .exit = cris_freq_cpu_exit, - .name = "cris_freq", - .owner = THIS_MODULE, - .attr = cris_freq_attr, -}; - -static int __init cris_freq_init(void) -{ - int ret; - ret = cpufreq_register_driver(&cris_freq_driver); - cpufreq_register_notifier(&cris_sdram_freq_notifier_block, - CPUFREQ_TRANSITION_NOTIFIER); - return ret; -} - -static int -cris_sdram_freq_notifier(struct notifier_block *nb, unsigned long val, - void *data) -{ - int i; - struct cpufreq_freqs *freqs = data; - if (val == CPUFREQ_PRECHANGE) { - reg_bif_core_rw_sdram_timing timing = - REG_RD(bif_core, regi_bif_core, rw_sdram_timing); - timing.cpd = (freqs->new == 200000 ? 0 : 1); - - if (freqs->new == 200000) - for (i = 0; i < 50000; i++) ; - REG_WR(bif_core, regi_bif_core, rw_sdram_timing, timing); - } - return 0; -} - -module_init(cris_freq_init); diff --git a/drivers/cpufreq/davinci-cpufreq.c b/drivers/cpufreq/davinci-cpufreq.c index 551dd655c6f2..2c277eb3795a 100644 --- a/drivers/cpufreq/davinci-cpufreq.c +++ b/drivers/cpufreq/davinci-cpufreq.c @@ -1,7 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * CPU frequency scaling for DaVinci * - * Copyright (C) 2009 Texas Instruments Incorporated - http://www.ti.com/ + * Copyright (C) 2009 Texas Instruments Incorporated - https://www.ti.com/ * * Based on linux/arch/arm/plat-omap/cpu-omap.c. Original Copyright follows: * @@ -13,23 +14,16 @@ * Copyright (C) 2007-2008 Texas Instruments, Inc. * Updated to support OMAP3 * Rajendra Nayak <rnayak@ti.com> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #include <linux/types.h> #include <linux/cpufreq.h> #include <linux/init.h> #include <linux/err.h> #include <linux/clk.h> +#include <linux/platform_data/davinci-cpufreq.h> #include <linux/platform_device.h> #include <linux/export.h> -#include <mach/hardware.h> -#include <mach/cpufreq.h> -#include <mach/common.h> - struct davinci_cpufreq { struct device *dev; struct clk *armclk; @@ -38,88 +32,38 @@ struct davinci_cpufreq { }; static struct davinci_cpufreq cpufreq; -static int davinci_verify_speed(struct cpufreq_policy *policy) +static int davinci_target(struct cpufreq_policy *policy, unsigned int idx) { struct davinci_cpufreq_config *pdata = cpufreq.dev->platform_data; - struct cpufreq_frequency_table *freq_table = pdata->freq_table; struct clk *armclk = cpufreq.armclk; - - if (freq_table) - return cpufreq_frequency_table_verify(policy, freq_table); - - if (policy->cpu) - return -EINVAL; - - cpufreq_verify_within_limits(policy, policy->cpuinfo.min_freq, - policy->cpuinfo.max_freq); - - policy->min = clk_round_rate(armclk, policy->min * 1000) / 1000; - policy->max = clk_round_rate(armclk, policy->max * 1000) / 1000; - cpufreq_verify_within_limits(policy, policy->cpuinfo.min_freq, - policy->cpuinfo.max_freq); - return 0; -} - -static unsigned int davinci_getspeed(unsigned int cpu) -{ - if (cpu) - return 0; - - return clk_get_rate(cpufreq.armclk) / 1000; -} - -static int davinci_target(struct cpufreq_policy *policy, - unsigned int target_freq, unsigned int relation) -{ + unsigned int old_freq, new_freq; int ret = 0; - unsigned int idx; - struct cpufreq_freqs freqs; - struct davinci_cpufreq_config *pdata = cpufreq.dev->platform_data; - struct clk *armclk = cpufreq.armclk; - - freqs.old = davinci_getspeed(0); - freqs.new = clk_round_rate(armclk, target_freq * 1000) / 1000; - - if (freqs.old == freqs.new) - return ret; - dev_dbg(cpufreq.dev, "transition: %u --> %u\n", freqs.old, freqs.new); - - ret = cpufreq_frequency_table_target(policy, pdata->freq_table, - freqs.new, relation, &idx); - if (ret) - return -EINVAL; - - cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE); + old_freq = policy->cur; + new_freq = pdata->freq_table[idx].frequency; /* if moving to higher frequency, up the voltage beforehand */ - if (pdata->set_voltage && freqs.new > freqs.old) { + if (pdata->set_voltage && new_freq > old_freq) { ret = pdata->set_voltage(idx); if (ret) - goto out; + return ret; } - ret = clk_set_rate(armclk, idx); + ret = clk_set_rate(armclk, new_freq * 1000); if (ret) - goto out; + return ret; if (cpufreq.asyncclk) { ret = clk_set_rate(cpufreq.asyncclk, cpufreq.asyncrate); if (ret) - goto out; + return ret; } /* if moving to lower freq, lower the voltage after lowering freq */ - if (pdata->set_voltage && freqs.new < freqs.old) + if (pdata->set_voltage && new_freq < old_freq) pdata->set_voltage(idx); -out: - if (ret) - freqs.new = freqs.old; - - cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); - - return ret; + return 0; } static int davinci_cpu_init(struct cpufreq_policy *policy) @@ -138,16 +82,7 @@ static int davinci_cpu_init(struct cpufreq_policy *policy) return result; } - policy->cur = davinci_getspeed(0); - - result = cpufreq_frequency_table_cpuinfo(policy, freq_table); - if (result) { - pr_err("%s: cpufreq_frequency_table_cpuinfo() failed", - __func__); - return result; - } - - cpufreq_frequency_table_get_attr(freq_table, policy->cpu); + policy->clk = cpufreq.armclk; /* * Time measurement across the target() function yields ~1500-1800us @@ -155,30 +90,17 @@ static int davinci_cpu_init(struct cpufreq_policy *policy) * Setting the latency to 2000 us to accommodate addition of drivers * to pre/post change notification list. */ - policy->cpuinfo.transition_latency = 2000 * 1000; - return 0; -} - -static int davinci_cpu_exit(struct cpufreq_policy *policy) -{ - cpufreq_frequency_table_put_attr(policy->cpu); + cpufreq_generic_init(policy, freq_table, 2000 * 1000); return 0; } -static struct freq_attr *davinci_cpufreq_attr[] = { - &cpufreq_freq_attr_scaling_available_freqs, - NULL, -}; - static struct cpufreq_driver davinci_driver = { - .flags = CPUFREQ_STICKY, - .verify = davinci_verify_speed, - .target = davinci_target, - .get = davinci_getspeed, + .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK, + .verify = cpufreq_generic_frequency_table_verify, + .target_index = davinci_target, + .get = cpufreq_generic_get, .init = davinci_cpu_init, - .exit = davinci_cpu_exit, .name = "davinci", - .attr = davinci_cpufreq_attr, }; static int __init davinci_cpufreq_probe(struct platform_device *pdev) @@ -208,20 +130,19 @@ static int __init davinci_cpufreq_probe(struct platform_device *pdev) return cpufreq_register_driver(&davinci_driver); } -static int __exit davinci_cpufreq_remove(struct platform_device *pdev) +static void __exit davinci_cpufreq_remove(struct platform_device *pdev) { + cpufreq_unregister_driver(&davinci_driver); + clk_put(cpufreq.armclk); if (cpufreq.asyncclk) clk_put(cpufreq.asyncclk); - - return cpufreq_unregister_driver(&davinci_driver); } static struct platform_driver davinci_cpufreq_driver = { .driver = { .name = "cpufreq-davinci", - .owner = THIS_MODULE, }, .remove = __exit_p(davinci_cpufreq_remove), }; diff --git a/drivers/cpufreq/dbx500-cpufreq.c b/drivers/cpufreq/dbx500-cpufreq.c deleted file mode 100644 index 1fdb02b9f1ec..000000000000 --- a/drivers/cpufreq/dbx500-cpufreq.c +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Copyright (C) STMicroelectronics 2009 - * Copyright (C) ST-Ericsson SA 2010-2012 - * - * License Terms: GNU General Public License v2 - * Author: Sundar Iyer <sundar.iyer@stericsson.com> - * Author: Martin Persson <martin.persson@stericsson.com> - * Author: Jonas Aaberg <jonas.aberg@stericsson.com> - */ - -#include <linux/module.h> -#include <linux/kernel.h> -#include <linux/cpufreq.h> -#include <linux/delay.h> -#include <linux/slab.h> -#include <linux/platform_device.h> -#include <linux/clk.h> - -static struct cpufreq_frequency_table *freq_table; -static struct clk *armss_clk; - -static struct freq_attr *dbx500_cpufreq_attr[] = { - &cpufreq_freq_attr_scaling_available_freqs, - NULL, -}; - -static int dbx500_cpufreq_verify_speed(struct cpufreq_policy *policy) -{ - return cpufreq_frequency_table_verify(policy, freq_table); -} - -static int dbx500_cpufreq_target(struct cpufreq_policy *policy, - unsigned int target_freq, - unsigned int relation) -{ - struct cpufreq_freqs freqs; - unsigned int idx; - int ret; - - /* Lookup the next frequency */ - if (cpufreq_frequency_table_target(policy, freq_table, target_freq, - relation, &idx)) - return -EINVAL; - - freqs.old = policy->cur; - freqs.new = freq_table[idx].frequency; - - if (freqs.old == freqs.new) - return 0; - - /* pre-change notification */ - cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE); - - /* update armss clk frequency */ - ret = clk_set_rate(armss_clk, freqs.new * 1000); - - if (ret) { - pr_err("dbx500-cpufreq: Failed to set armss_clk to %d Hz: error %d\n", - freqs.new * 1000, ret); - freqs.new = freqs.old; - } - - /* post change notification */ - cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); - - return ret; -} - -static unsigned int dbx500_cpufreq_getspeed(unsigned int cpu) -{ - int i = 0; - unsigned long freq = clk_get_rate(armss_clk) / 1000; - - /* The value is rounded to closest frequency in the defined table. */ - while (freq_table[i + 1].frequency != CPUFREQ_TABLE_END) { - if (freq < freq_table[i].frequency + - (freq_table[i + 1].frequency - freq_table[i].frequency) / 2) - return freq_table[i].frequency; - i++; - } - - return freq_table[i].frequency; -} - -static int __cpuinit dbx500_cpufreq_init(struct cpufreq_policy *policy) -{ - int res; - - /* get policy fields based on the table */ - res = cpufreq_frequency_table_cpuinfo(policy, freq_table); - if (!res) - cpufreq_frequency_table_get_attr(freq_table, policy->cpu); - else { - pr_err("dbx500-cpufreq: Failed to read policy table\n"); - return res; - } - - policy->min = policy->cpuinfo.min_freq; - policy->max = policy->cpuinfo.max_freq; - policy->cur = dbx500_cpufreq_getspeed(policy->cpu); - policy->governor = CPUFREQ_DEFAULT_GOVERNOR; - - /* - * FIXME : Need to take time measurement across the target() - * function with no/some/all drivers in the notification - * list. - */ - policy->cpuinfo.transition_latency = 20 * 1000; /* in ns */ - - /* policy sharing between dual CPUs */ - cpumask_setall(policy->cpus); - - return 0; -} - -static struct cpufreq_driver dbx500_cpufreq_driver = { - .flags = CPUFREQ_STICKY | CPUFREQ_CONST_LOOPS, - .verify = dbx500_cpufreq_verify_speed, - .target = dbx500_cpufreq_target, - .get = dbx500_cpufreq_getspeed, - .init = dbx500_cpufreq_init, - .name = "DBX500", - .attr = dbx500_cpufreq_attr, -}; - -static int dbx500_cpufreq_probe(struct platform_device *pdev) -{ - int i = 0; - - freq_table = dev_get_platdata(&pdev->dev); - if (!freq_table) { - pr_err("dbx500-cpufreq: Failed to fetch cpufreq table\n"); - return -ENODEV; - } - - armss_clk = clk_get(&pdev->dev, "armss"); - if (IS_ERR(armss_clk)) { - pr_err("dbx500-cpufreq: Failed to get armss clk\n"); - return PTR_ERR(armss_clk); - } - - pr_info("dbx500-cpufreq: Available frequencies:\n"); - while (freq_table[i].frequency != CPUFREQ_TABLE_END) { - pr_info(" %d Mhz\n", freq_table[i].frequency/1000); - i++; - } - - return cpufreq_register_driver(&dbx500_cpufreq_driver); -} - -static struct platform_driver dbx500_cpufreq_plat_driver = { - .driver = { - .name = "cpufreq-ux500", - .owner = THIS_MODULE, - }, - .probe = dbx500_cpufreq_probe, -}; - -static int __init dbx500_cpufreq_register(void) -{ - return platform_driver_register(&dbx500_cpufreq_plat_driver); -} -device_initcall(dbx500_cpufreq_register); - -MODULE_LICENSE("GPL v2"); -MODULE_DESCRIPTION("cpufreq driver for DBX500"); diff --git a/drivers/cpufreq/e_powersaver.c b/drivers/cpufreq/e_powersaver.c index a60efaeb4cf8..320a0af2266a 100644 --- a/drivers/cpufreq/e_powersaver.c +++ b/drivers/cpufreq/e_powersaver.c @@ -1,11 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Based on documentation provided by Dave Jones. Thanks! * - * Licensed under the terms of the GNU GPL License version 2. - * * BIG FAT DISCLAIMER: Work in progress code. Possibly *dangerous* */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + #include <linux/kernel.h> #include <linux/module.h> #include <linux/init.h> @@ -20,7 +21,7 @@ #include <asm/msr.h> #include <asm/tsc.h> -#if defined CONFIG_ACPI_PROCESSOR || defined CONFIG_ACPI_PROCESSOR_MODULE +#if IS_ENABLED(CONFIG_ACPI_PROCESSOR) #include <linux/acpi.h> #include <acpi/processor.h> #endif @@ -33,7 +34,7 @@ struct eps_cpu_data { u32 fsb; -#if defined CONFIG_ACPI_PROCESSOR || defined CONFIG_ACPI_PROCESSOR_MODULE +#if IS_ENABLED(CONFIG_ACPI_PROCESSOR) u32 bios_limit; #endif struct cpufreq_frequency_table freq_table[]; @@ -46,7 +47,7 @@ static int freq_failsafe_off; static int voltage_failsafe_off; static int set_max_voltage; -#if defined CONFIG_ACPI_PROCESSOR || defined CONFIG_ACPI_PROCESSOR_MODULE +#if IS_ENABLED(CONFIG_ACPI_PROCESSOR) static int ignore_acpi_limit; static struct acpi_processor_performance *eps_acpi_cpu_perf; @@ -54,7 +55,7 @@ static struct acpi_processor_performance *eps_acpi_cpu_perf; /* Minimum necessary to get acpi_processor_get_bios_limit() working */ static int eps_acpi_init(void) { - eps_acpi_cpu_perf = kzalloc(sizeof(struct acpi_processor_performance), + eps_acpi_cpu_perf = kzalloc(sizeof(*eps_acpi_cpu_perf), GFP_KERNEL); if (!eps_acpi_cpu_perf) return -ENOMEM; @@ -78,7 +79,7 @@ static int eps_acpi_init(void) static int eps_acpi_exit(struct cpufreq_policy *policy) { if (eps_acpi_cpu_perf) { - acpi_processor_unregister_performance(eps_acpi_cpu_perf, 0); + acpi_processor_unregister_performance(0); free_cpumask_var(eps_acpi_cpu_perf->shared_cpu_map); kfree(eps_acpi_cpu_perf); eps_acpi_cpu_perf = NULL; @@ -107,15 +108,9 @@ static int eps_set_state(struct eps_cpu_data *centaur, struct cpufreq_policy *policy, u32 dest_state) { - struct cpufreq_freqs freqs; u32 lo, hi; - int err = 0; int i; - freqs.old = eps_get(policy->cpu); - freqs.new = centaur->fsb * ((dest_state >> 8) & 0xff); - cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE); - /* Wait while CPU is busy */ rdmsr(MSR_IA32_PERF_STATUS, lo, hi); i = 0; @@ -124,8 +119,7 @@ static int eps_set_state(struct eps_cpu_data *centaur, rdmsr(MSR_IA32_PERF_STATUS, lo, hi); i++; if (unlikely(i > 64)) { - err = -ENODEV; - goto postchange; + return -ENODEV; } } /* Set new multiplier and voltage */ @@ -137,16 +131,10 @@ static int eps_set_state(struct eps_cpu_data *centaur, rdmsr(MSR_IA32_PERF_STATUS, lo, hi); i++; if (unlikely(i > 64)) { - err = -ENODEV; - goto postchange; + return -ENODEV; } } while (lo & ((1 << 16) | (1 << 17))); - /* Return current frequency */ -postchange: - rdmsr(MSR_IA32_PERF_STATUS, lo, hi); - freqs.new = centaur->fsb * ((lo >> 8) & 0xff); - #ifdef DEBUG { u8 current_multiplier, current_voltage; @@ -154,26 +142,17 @@ postchange: /* Print voltage and multiplier */ rdmsr(MSR_IA32_PERF_STATUS, lo, hi); current_voltage = lo & 0xff; - printk(KERN_INFO "eps: Current voltage = %dmV\n", - current_voltage * 16 + 700); + pr_info("Current voltage = %dmV\n", current_voltage * 16 + 700); current_multiplier = (lo >> 8) & 0xff; - printk(KERN_INFO "eps: Current multiplier = %d\n", - current_multiplier); + pr_info("Current multiplier = %d\n", current_multiplier); } #endif - if (err) - freqs.new = freqs.old; - - cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); - return err; + return 0; } -static int eps_target(struct cpufreq_policy *policy, - unsigned int target_freq, - unsigned int relation) +static int eps_target(struct cpufreq_policy *policy, unsigned int index) { struct eps_cpu_data *centaur; - unsigned int newstate = 0; unsigned int cpu = policy->cpu; unsigned int dest_state; int ret; @@ -182,28 +161,14 @@ static int eps_target(struct cpufreq_policy *policy, return -ENODEV; centaur = eps_cpu[cpu]; - if (unlikely(cpufreq_frequency_table_target(policy, - &eps_cpu[cpu]->freq_table[0], - target_freq, - relation, - &newstate))) { - return -EINVAL; - } - /* Make frequency transition */ - dest_state = centaur->freq_table[newstate].driver_data & 0xffff; + dest_state = centaur->freq_table[index].driver_data & 0xffff; ret = eps_set_state(centaur, policy, dest_state); if (ret) - printk(KERN_ERR "eps: Timeout!\n"); + pr_err("Timeout!\n"); return ret; } -static int eps_verify(struct cpufreq_policy *policy) -{ - return cpufreq_frequency_table_verify(policy, - &eps_cpu[policy->cpu]->freq_table[0]); -} - static int eps_cpu_init(struct cpufreq_policy *policy) { unsigned int i; @@ -218,9 +183,8 @@ static int eps_cpu_init(struct cpufreq_policy *policy) struct cpuinfo_x86 *c = &cpu_data(0); struct cpufreq_frequency_table *f_table; int k, step, voltage; - int ret; int states; -#if defined CONFIG_ACPI_PROCESSOR || defined CONFIG_ACPI_PROCESSOR_MODULE +#if IS_ENABLED(CONFIG_ACPI_PROCESSOR) unsigned int limit; #endif @@ -228,48 +192,47 @@ static int eps_cpu_init(struct cpufreq_policy *policy) return -ENODEV; /* Check brand */ - printk(KERN_INFO "eps: Detected VIA "); + pr_info("Detected VIA "); switch (c->x86_model) { case 10: rdmsr(0x1153, lo, hi); brand = (((lo >> 2) ^ lo) >> 18) & 3; - printk(KERN_CONT "Model A "); + pr_cont("Model A "); break; case 13: rdmsr(0x1154, lo, hi); brand = (((lo >> 4) ^ (lo >> 2))) & 0x000000ff; - printk(KERN_CONT "Model D "); + pr_cont("Model D "); break; } switch (brand) { case EPS_BRAND_C7M: - printk(KERN_CONT "C7-M\n"); + pr_cont("C7-M\n"); break; case EPS_BRAND_C7: - printk(KERN_CONT "C7\n"); + pr_cont("C7\n"); break; case EPS_BRAND_EDEN: - printk(KERN_CONT "Eden\n"); + pr_cont("Eden\n"); break; case EPS_BRAND_C7D: - printk(KERN_CONT "C7-D\n"); + pr_cont("C7-D\n"); break; case EPS_BRAND_C3: - printk(KERN_CONT "C3\n"); + pr_cont("C3\n"); return -ENODEV; - break; } /* Enable Enhanced PowerSaver */ - rdmsrl(MSR_IA32_MISC_ENABLE, val); + rdmsrq(MSR_IA32_MISC_ENABLE, val); if (!(val & MSR_IA32_MISC_ENABLE_ENHANCED_SPEEDSTEP)) { val |= MSR_IA32_MISC_ENABLE_ENHANCED_SPEEDSTEP; - wrmsrl(MSR_IA32_MISC_ENABLE, val); + wrmsrq(MSR_IA32_MISC_ENABLE, val); /* Can be locked at 0 */ - rdmsrl(MSR_IA32_MISC_ENABLE, val); + rdmsrq(MSR_IA32_MISC_ENABLE, val); if (!(val & MSR_IA32_MISC_ENABLE_ENHANCED_SPEEDSTEP)) { - printk(KERN_INFO "eps: Can't enable Enhanced PowerSaver\n"); + pr_info("Can't enable Enhanced PowerSaver\n"); return -ENODEV; } } @@ -277,22 +240,19 @@ static int eps_cpu_init(struct cpufreq_policy *policy) /* Print voltage and multiplier */ rdmsr(MSR_IA32_PERF_STATUS, lo, hi); current_voltage = lo & 0xff; - printk(KERN_INFO "eps: Current voltage = %dmV\n", - current_voltage * 16 + 700); + pr_info("Current voltage = %dmV\n", current_voltage * 16 + 700); current_multiplier = (lo >> 8) & 0xff; - printk(KERN_INFO "eps: Current multiplier = %d\n", current_multiplier); + pr_info("Current multiplier = %d\n", current_multiplier); /* Print limits */ max_voltage = hi & 0xff; - printk(KERN_INFO "eps: Highest voltage = %dmV\n", - max_voltage * 16 + 700); + pr_info("Highest voltage = %dmV\n", max_voltage * 16 + 700); max_multiplier = (hi >> 8) & 0xff; - printk(KERN_INFO "eps: Highest multiplier = %d\n", max_multiplier); + pr_info("Highest multiplier = %d\n", max_multiplier); min_voltage = (hi >> 16) & 0xff; - printk(KERN_INFO "eps: Lowest voltage = %dmV\n", - min_voltage * 16 + 700); + pr_info("Lowest voltage = %dmV\n", min_voltage * 16 + 700); min_multiplier = (hi >> 24) & 0xff; - printk(KERN_INFO "eps: Lowest multiplier = %d\n", min_multiplier); + pr_info("Lowest multiplier = %d\n", min_multiplier); /* Sanity checks */ if (current_multiplier == 0 || max_multiplier == 0 @@ -310,34 +270,30 @@ static int eps_cpu_init(struct cpufreq_policy *policy) /* Check for systems using underclocked CPU */ if (!freq_failsafe_off && max_multiplier != current_multiplier) { - printk(KERN_INFO "eps: Your processor is running at different " - "frequency then its maximum. Aborting.\n"); - printk(KERN_INFO "eps: You can use freq_failsafe_off option " - "to disable this check.\n"); + pr_info("Your processor is running at different frequency then its maximum. Aborting.\n"); + pr_info("You can use freq_failsafe_off option to disable this check.\n"); return -EINVAL; } if (!voltage_failsafe_off && max_voltage != current_voltage) { - printk(KERN_INFO "eps: Your processor is running at different " - "voltage then its maximum. Aborting.\n"); - printk(KERN_INFO "eps: You can use voltage_failsafe_off " - "option to disable this check.\n"); + pr_info("Your processor is running at different voltage then its maximum. Aborting.\n"); + pr_info("You can use voltage_failsafe_off option to disable this check.\n"); return -EINVAL; } /* Calc FSB speed */ fsb = cpu_khz / current_multiplier; -#if defined CONFIG_ACPI_PROCESSOR || defined CONFIG_ACPI_PROCESSOR_MODULE +#if IS_ENABLED(CONFIG_ACPI_PROCESSOR) /* Check for ACPI processor speed limit */ if (!ignore_acpi_limit && !eps_acpi_init()) { if (!acpi_processor_get_bios_limit(policy->cpu, &limit)) { - printk(KERN_INFO "eps: ACPI limit %u.%uGHz\n", + pr_info("ACPI limit %u.%uGHz\n", limit/1000000, (limit%1000000)/10000); eps_acpi_exit(policy); /* Check if max_multiplier is in BIOS limits */ if (limit && max_multiplier * fsb > limit) { - printk(KERN_INFO "eps: Aborting.\n"); + pr_info("Aborting\n"); return -EINVAL; } } @@ -353,8 +309,7 @@ static int eps_cpu_init(struct cpufreq_policy *policy) v = (set_max_voltage - 700) / 16; /* Check if voltage is within limits */ if (v >= min_voltage && v <= max_voltage) { - printk(KERN_INFO "eps: Setting %dmV as maximum.\n", - v * 16 + 700); + pr_info("Setting %dmV as maximum\n", v * 16 + 700); max_voltage = v; } } @@ -366,16 +321,15 @@ static int eps_cpu_init(struct cpufreq_policy *policy) states = 2; /* Allocate private data and frequency table for current cpu */ - centaur = kzalloc(sizeof(struct eps_cpu_data) - + (states + 1) * sizeof(struct cpufreq_frequency_table), - GFP_KERNEL); + centaur = kzalloc(struct_size(centaur, freq_table, states + 1), + GFP_KERNEL); if (!centaur) return -ENOMEM; eps_cpu[0] = centaur; /* Copy basic values */ centaur->fsb = fsb; -#if defined CONFIG_ACPI_PROCESSOR || defined CONFIG_ACPI_PROCESSOR_MODULE +#if IS_ENABLED(CONFIG_ACPI_PROCESSOR) centaur->bios_limit = limit; #endif @@ -401,50 +355,34 @@ static int eps_cpu_init(struct cpufreq_policy *policy) } policy->cpuinfo.transition_latency = 140000; /* 844mV -> 700mV in ns */ - policy->cur = fsb * current_multiplier; + policy->freq_table = ¢aur->freq_table[0]; - ret = cpufreq_frequency_table_cpuinfo(policy, ¢aur->freq_table[0]); - if (ret) { - kfree(centaur); - return ret; - } - - cpufreq_frequency_table_get_attr(¢aur->freq_table[0], policy->cpu); return 0; } -static int eps_cpu_exit(struct cpufreq_policy *policy) +static void eps_cpu_exit(struct cpufreq_policy *policy) { unsigned int cpu = policy->cpu; /* Bye */ - cpufreq_frequency_table_put_attr(policy->cpu); kfree(eps_cpu[cpu]); eps_cpu[cpu] = NULL; - return 0; } -static struct freq_attr *eps_attr[] = { - &cpufreq_freq_attr_scaling_available_freqs, - NULL, -}; - static struct cpufreq_driver eps_driver = { - .verify = eps_verify, - .target = eps_target, + .verify = cpufreq_generic_frequency_table_verify, + .target_index = eps_target, .init = eps_cpu_init, .exit = eps_cpu_exit, .get = eps_get, .name = "e_powersaver", - .owner = THIS_MODULE, - .attr = eps_attr, }; /* This driver will work only on Centaur C7 processors with * Enhanced SpeedStep/PowerSaver registers */ static const struct x86_cpu_id eps_cpu_id[] = { - { X86_VENDOR_CENTAUR, 6, X86_MODEL_ANY, X86_FEATURE_EST }, + X86_MATCH_VENDOR_FAM_FEATURE(CENTAUR, 6, X86_FEATURE_EST, NULL), {} }; MODULE_DEVICE_TABLE(x86cpu, eps_cpu_id); @@ -469,7 +407,7 @@ module_param(freq_failsafe_off, int, 0644); MODULE_PARM_DESC(freq_failsafe_off, "Disable current vs max frequency check"); module_param(voltage_failsafe_off, int, 0644); MODULE_PARM_DESC(voltage_failsafe_off, "Disable current vs max voltage check"); -#if defined CONFIG_ACPI_PROCESSOR || defined CONFIG_ACPI_PROCESSOR_MODULE +#if IS_ENABLED(CONFIG_ACPI_PROCESSOR) module_param(ignore_acpi_limit, int, 0644); MODULE_PARM_DESC(ignore_acpi_limit, "Don't check ACPI's processor speed limit"); #endif diff --git a/drivers/cpufreq/elanfreq.c b/drivers/cpufreq/elanfreq.c index 658d860344b0..fc5a58088b35 100644 --- a/drivers/cpufreq/elanfreq.c +++ b/drivers/cpufreq/elanfreq.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-or-later /* * elanfreq: cpufreq driver for the AMD ELAN family * @@ -7,15 +8,11 @@ * * All Rights Reserved. * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - * * 2002-02-13: - initial revision for 2.4.18-pre9 by Robert Schwebel - * */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + #include <linux/kernel.h> #include <linux/module.h> #include <linux/init.h> @@ -24,7 +21,6 @@ #include <linux/cpufreq.h> #include <asm/cpu_device_id.h> -#include <asm/msr.h> #include <linux/timex.h> #include <linux/io.h> @@ -56,15 +52,15 @@ static struct s_elan_multiplier elan_multiplier[] = { }; static struct cpufreq_frequency_table elanfreq_table[] = { - {0, 1000}, - {1, 2000}, - {2, 4000}, - {3, 8000}, - {4, 16000}, - {5, 33000}, - {6, 66000}, - {7, 99000}, - {0, CPUFREQ_TABLE_END}, + {0, 0, 1000}, + {0, 1, 2000}, + {0, 2, 4000}, + {0, 3, 8000}, + {0, 4, 16000}, + {0, 5, 33000}, + {0, 6, 66000}, + {0, 7, 99000}, + {0, 0, CPUFREQ_TABLE_END}, }; @@ -105,32 +101,9 @@ static unsigned int elanfreq_get_cpu_frequency(unsigned int cpu) } -/** - * elanfreq_set_cpu_frequency: Change the CPU core frequency - * @cpu: cpu number - * @freq: frequency in kHz - * - * This function takes a frequency value and changes the CPU frequency - * according to this. Note that the frequency has to be checked by - * elanfreq_validatespeed() for correctness! - * - * There is no return value. - */ - -static void elanfreq_set_cpu_state(struct cpufreq_policy *policy, - unsigned int state) +static int elanfreq_target(struct cpufreq_policy *policy, + unsigned int state) { - struct cpufreq_freqs freqs; - - freqs.old = elanfreq_get_cpu_frequency(0); - freqs.new = elan_multiplier[state].clock; - - cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE); - - printk(KERN_INFO "elanfreq: attempting to set frequency to %i kHz\n", - elan_multiplier[state].clock); - - /* * Access to the Elan's internal registers is indexed via * 0x22: Chip Setup & Control Register Index Register (CSCI) @@ -161,39 +134,8 @@ static void elanfreq_set_cpu_state(struct cpufreq_policy *policy, udelay(10000); local_irq_enable(); - cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); -}; - - -/** - * elanfreq_validatespeed: test if frequency range is valid - * @policy: the policy to validate - * - * This function checks if a given frequency range in kHz is valid - * for the hardware supported by the driver. - */ - -static int elanfreq_verify(struct cpufreq_policy *policy) -{ - return cpufreq_frequency_table_verify(policy, &elanfreq_table[0]); -} - -static int elanfreq_target(struct cpufreq_policy *policy, - unsigned int target_freq, - unsigned int relation) -{ - unsigned int newstate = 0; - - if (cpufreq_frequency_table_target(policy, &elanfreq_table[0], - target_freq, relation, &newstate)) - return -EINVAL; - - elanfreq_set_cpu_state(policy, newstate); - return 0; } - - /* * Module init and exit code */ @@ -201,8 +143,7 @@ static int elanfreq_target(struct cpufreq_policy *policy, static int elanfreq_cpu_init(struct cpufreq_policy *policy) { struct cpuinfo_x86 *c = &cpu_data(0); - unsigned int i; - int result; + struct cpufreq_frequency_table *pos; /* capability check */ if ((c->x86_vendor != X86_VENDOR_AMD) || @@ -214,27 +155,11 @@ static int elanfreq_cpu_init(struct cpufreq_policy *policy) max_freq = elanfreq_get_cpu_frequency(0); /* table init */ - for (i = 0; (elanfreq_table[i].frequency != CPUFREQ_TABLE_END); i++) { - if (elanfreq_table[i].frequency > max_freq) - elanfreq_table[i].frequency = CPUFREQ_ENTRY_INVALID; - } + cpufreq_for_each_entry(pos, elanfreq_table) + if (pos->frequency > max_freq) + pos->frequency = CPUFREQ_ENTRY_INVALID; - /* cpuinfo and default policy values */ - policy->cpuinfo.transition_latency = CPUFREQ_ETERNAL; - policy->cur = elanfreq_get_cpu_frequency(0); - - result = cpufreq_frequency_table_cpuinfo(policy, elanfreq_table); - if (result) - return result; - - cpufreq_frequency_table_get_attr(elanfreq_table, policy->cpu); - return 0; -} - - -static int elanfreq_cpu_exit(struct cpufreq_policy *policy) -{ - cpufreq_frequency_table_put_attr(policy->cpu); + policy->freq_table = elanfreq_table; return 0; } @@ -254,32 +179,24 @@ static int elanfreq_cpu_exit(struct cpufreq_policy *policy) static int __init elanfreq_setup(char *str) { max_freq = simple_strtoul(str, &str, 0); - printk(KERN_WARNING "You're using the deprecated elanfreq command line option. Use elanfreq.max_freq instead, please!\n"); + pr_warn("You're using the deprecated elanfreq command line option. Use elanfreq.max_freq instead, please!\n"); return 1; } __setup("elanfreq=", elanfreq_setup); #endif -static struct freq_attr *elanfreq_attr[] = { - &cpufreq_freq_attr_scaling_available_freqs, - NULL, -}; - - static struct cpufreq_driver elanfreq_driver = { .get = elanfreq_get_cpu_frequency, - .verify = elanfreq_verify, - .target = elanfreq_target, + .flags = CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING, + .verify = cpufreq_generic_frequency_table_verify, + .target_index = elanfreq_target, .init = elanfreq_cpu_init, - .exit = elanfreq_cpu_exit, .name = "elanfreq", - .owner = THIS_MODULE, - .attr = elanfreq_attr, }; static const struct x86_cpu_id elan_id[] = { - { X86_VENDOR_AMD, 4, 10, }, + X86_MATCH_VENDOR_FAM_MODEL(AMD, 4, 10, NULL), {} }; MODULE_DEVICE_TABLE(x86cpu, elan_id); diff --git a/drivers/cpufreq/exynos-cpufreq.c b/drivers/cpufreq/exynos-cpufreq.c deleted file mode 100644 index 0d32f02ef4d6..000000000000 --- a/drivers/cpufreq/exynos-cpufreq.c +++ /dev/null @@ -1,338 +0,0 @@ -/* - * Copyright (c) 2010-2011 Samsung Electronics Co., Ltd. - * http://www.samsung.com - * - * EXYNOS - CPU frequency scaling support for EXYNOS series - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. -*/ - -#include <linux/kernel.h> -#include <linux/err.h> -#include <linux/clk.h> -#include <linux/io.h> -#include <linux/slab.h> -#include <linux/regulator/consumer.h> -#include <linux/cpufreq.h> -#include <linux/suspend.h> - -#include <plat/cpu.h> - -#include "exynos-cpufreq.h" - -static struct exynos_dvfs_info *exynos_info; - -static struct regulator *arm_regulator; -static struct cpufreq_freqs freqs; - -static unsigned int locking_frequency; -static bool frequency_locked; -static DEFINE_MUTEX(cpufreq_lock); - -static int exynos_verify_speed(struct cpufreq_policy *policy) -{ - return cpufreq_frequency_table_verify(policy, - exynos_info->freq_table); -} - -static unsigned int exynos_getspeed(unsigned int cpu) -{ - return clk_get_rate(exynos_info->cpu_clk) / 1000; -} - -static int exynos_cpufreq_get_index(unsigned int freq) -{ - struct cpufreq_frequency_table *freq_table = exynos_info->freq_table; - int index; - - for (index = 0; - freq_table[index].frequency != CPUFREQ_TABLE_END; index++) - if (freq_table[index].frequency == freq) - break; - - if (freq_table[index].frequency == CPUFREQ_TABLE_END) - return -EINVAL; - - return index; -} - -static int exynos_cpufreq_scale(unsigned int target_freq) -{ - struct cpufreq_frequency_table *freq_table = exynos_info->freq_table; - unsigned int *volt_table = exynos_info->volt_table; - struct cpufreq_policy *policy = cpufreq_cpu_get(0); - unsigned int arm_volt, safe_arm_volt = 0; - unsigned int mpll_freq_khz = exynos_info->mpll_freq_khz; - int index, old_index; - int ret = 0; - - freqs.old = policy->cur; - freqs.new = target_freq; - - if (freqs.new == freqs.old) - goto out; - - /* - * The policy max have been changed so that we cannot get proper - * old_index with cpufreq_frequency_table_target(). Thus, ignore - * policy and get the index from the raw freqeuncy table. - */ - old_index = exynos_cpufreq_get_index(freqs.old); - if (old_index < 0) { - ret = old_index; - goto out; - } - - index = exynos_cpufreq_get_index(target_freq); - if (index < 0) { - ret = index; - goto out; - } - - /* - * ARM clock source will be changed APLL to MPLL temporary - * To support this level, need to control regulator for - * required voltage level - */ - if (exynos_info->need_apll_change != NULL) { - if (exynos_info->need_apll_change(old_index, index) && - (freq_table[index].frequency < mpll_freq_khz) && - (freq_table[old_index].frequency < mpll_freq_khz)) - safe_arm_volt = volt_table[exynos_info->pll_safe_idx]; - } - arm_volt = volt_table[index]; - - cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE); - - /* When the new frequency is higher than current frequency */ - if ((freqs.new > freqs.old) && !safe_arm_volt) { - /* Firstly, voltage up to increase frequency */ - ret = regulator_set_voltage(arm_regulator, arm_volt, arm_volt); - if (ret) { - pr_err("%s: failed to set cpu voltage to %d\n", - __func__, arm_volt); - freqs.new = freqs.old; - goto post_notify; - } - } - - if (safe_arm_volt) { - ret = regulator_set_voltage(arm_regulator, safe_arm_volt, - safe_arm_volt); - if (ret) { - pr_err("%s: failed to set cpu voltage to %d\n", - __func__, safe_arm_volt); - freqs.new = freqs.old; - goto post_notify; - } - } - - exynos_info->set_freq(old_index, index); - -post_notify: - cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); - - if (ret) - goto out; - - /* When the new frequency is lower than current frequency */ - if ((freqs.new < freqs.old) || - ((freqs.new > freqs.old) && safe_arm_volt)) { - /* down the voltage after frequency change */ - regulator_set_voltage(arm_regulator, arm_volt, - arm_volt); - if (ret) { - pr_err("%s: failed to set cpu voltage to %d\n", - __func__, arm_volt); - goto out; - } - } - -out: - - cpufreq_cpu_put(policy); - - return ret; -} - -static int exynos_target(struct cpufreq_policy *policy, - unsigned int target_freq, - unsigned int relation) -{ - struct cpufreq_frequency_table *freq_table = exynos_info->freq_table; - unsigned int index; - unsigned int new_freq; - int ret = 0; - - mutex_lock(&cpufreq_lock); - - if (frequency_locked) - goto out; - - if (cpufreq_frequency_table_target(policy, freq_table, - target_freq, relation, &index)) { - ret = -EINVAL; - goto out; - } - - new_freq = freq_table[index].frequency; - - ret = exynos_cpufreq_scale(new_freq); - -out: - mutex_unlock(&cpufreq_lock); - - return ret; -} - -#ifdef CONFIG_PM -static int exynos_cpufreq_suspend(struct cpufreq_policy *policy) -{ - return 0; -} - -static int exynos_cpufreq_resume(struct cpufreq_policy *policy) -{ - return 0; -} -#endif - -/** - * exynos_cpufreq_pm_notifier - block CPUFREQ's activities in suspend-resume - * context - * @notifier - * @pm_event - * @v - * - * While frequency_locked == true, target() ignores every frequency but - * locking_frequency. The locking_frequency value is the initial frequency, - * which is set by the bootloader. In order to eliminate possible - * inconsistency in clock values, we save and restore frequencies during - * suspend and resume and block CPUFREQ activities. Note that the standard - * suspend/resume cannot be used as they are too deep (syscore_ops) for - * regulator actions. - */ -static int exynos_cpufreq_pm_notifier(struct notifier_block *notifier, - unsigned long pm_event, void *v) -{ - int ret; - - switch (pm_event) { - case PM_SUSPEND_PREPARE: - mutex_lock(&cpufreq_lock); - frequency_locked = true; - mutex_unlock(&cpufreq_lock); - - ret = exynos_cpufreq_scale(locking_frequency); - if (ret < 0) - return NOTIFY_BAD; - - break; - - case PM_POST_SUSPEND: - mutex_lock(&cpufreq_lock); - frequency_locked = false; - mutex_unlock(&cpufreq_lock); - break; - } - - return NOTIFY_OK; -} - -static struct notifier_block exynos_cpufreq_nb = { - .notifier_call = exynos_cpufreq_pm_notifier, -}; - -static int exynos_cpufreq_cpu_init(struct cpufreq_policy *policy) -{ - policy->cur = policy->min = policy->max = exynos_getspeed(policy->cpu); - - cpufreq_frequency_table_get_attr(exynos_info->freq_table, policy->cpu); - - /* set the transition latency value */ - policy->cpuinfo.transition_latency = 100000; - - cpumask_setall(policy->cpus); - - return cpufreq_frequency_table_cpuinfo(policy, exynos_info->freq_table); -} - -static int exynos_cpufreq_cpu_exit(struct cpufreq_policy *policy) -{ - cpufreq_frequency_table_put_attr(policy->cpu); - return 0; -} - -static struct freq_attr *exynos_cpufreq_attr[] = { - &cpufreq_freq_attr_scaling_available_freqs, - NULL, -}; - -static struct cpufreq_driver exynos_driver = { - .flags = CPUFREQ_STICKY, - .verify = exynos_verify_speed, - .target = exynos_target, - .get = exynos_getspeed, - .init = exynos_cpufreq_cpu_init, - .exit = exynos_cpufreq_cpu_exit, - .name = "exynos_cpufreq", - .attr = exynos_cpufreq_attr, -#ifdef CONFIG_PM - .suspend = exynos_cpufreq_suspend, - .resume = exynos_cpufreq_resume, -#endif -}; - -static int __init exynos_cpufreq_init(void) -{ - int ret = -EINVAL; - - exynos_info = kzalloc(sizeof(struct exynos_dvfs_info), GFP_KERNEL); - if (!exynos_info) - return -ENOMEM; - - if (soc_is_exynos4210()) - ret = exynos4210_cpufreq_init(exynos_info); - else if (soc_is_exynos4212() || soc_is_exynos4412()) - ret = exynos4x12_cpufreq_init(exynos_info); - else if (soc_is_exynos5250()) - ret = exynos5250_cpufreq_init(exynos_info); - else - return 0; - - if (ret) - goto err_vdd_arm; - - if (exynos_info->set_freq == NULL) { - pr_err("%s: No set_freq function (ERR)\n", __func__); - goto err_vdd_arm; - } - - arm_regulator = regulator_get(NULL, "vdd_arm"); - if (IS_ERR(arm_regulator)) { - pr_err("%s: failed to get resource vdd_arm\n", __func__); - goto err_vdd_arm; - } - - locking_frequency = exynos_getspeed(0); - - register_pm_notifier(&exynos_cpufreq_nb); - - if (cpufreq_register_driver(&exynos_driver)) { - pr_err("%s: failed to register cpufreq driver\n", __func__); - goto err_cpufreq; - } - - return 0; -err_cpufreq: - unregister_pm_notifier(&exynos_cpufreq_nb); - - regulator_put(arm_regulator); -err_vdd_arm: - kfree(exynos_info); - pr_debug("%s: failed initialization\n", __func__); - return -EINVAL; -} -late_initcall(exynos_cpufreq_init); diff --git a/drivers/cpufreq/exynos-cpufreq.h b/drivers/cpufreq/exynos-cpufreq.h deleted file mode 100644 index 92b852ee5ddc..000000000000 --- a/drivers/cpufreq/exynos-cpufreq.h +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright (c) 2010 Samsung Electronics Co., Ltd. - * http://www.samsung.com - * - * EXYNOS - CPUFreq support - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. -*/ - -enum cpufreq_level_index { - L0, L1, L2, L3, L4, - L5, L6, L7, L8, L9, - L10, L11, L12, L13, L14, - L15, L16, L17, L18, L19, - L20, -}; - -#define APLL_FREQ(f, a0, a1, a2, a3, a4, a5, a6, a7, b0, b1, b2, m, p, s) \ - { \ - .freq = (f) * 1000, \ - .clk_div_cpu0 = ((a0) | (a1) << 4 | (a2) << 8 | (a3) << 12 | \ - (a4) << 16 | (a5) << 20 | (a6) << 24 | (a7) << 28), \ - .clk_div_cpu1 = (b0 << 0 | b1 << 4 | b2 << 8), \ - .mps = ((m) << 16 | (p) << 8 | (s)), \ - } - -struct apll_freq { - unsigned int freq; - u32 clk_div_cpu0; - u32 clk_div_cpu1; - u32 mps; -}; - -struct exynos_dvfs_info { - unsigned long mpll_freq_khz; - unsigned int pll_safe_idx; - struct clk *cpu_clk; - unsigned int *volt_table; - struct cpufreq_frequency_table *freq_table; - void (*set_freq)(unsigned int, unsigned int); - bool (*need_apll_change)(unsigned int, unsigned int); -}; - -extern int exynos4210_cpufreq_init(struct exynos_dvfs_info *); -extern int exynos4x12_cpufreq_init(struct exynos_dvfs_info *); -extern int exynos5250_cpufreq_init(struct exynos_dvfs_info *); diff --git a/drivers/cpufreq/exynos4210-cpufreq.c b/drivers/cpufreq/exynos4210-cpufreq.c deleted file mode 100644 index add7fbec4fc9..000000000000 --- a/drivers/cpufreq/exynos4210-cpufreq.c +++ /dev/null @@ -1,211 +0,0 @@ -/* - * Copyright (c) 2010-2011 Samsung Electronics Co., Ltd. - * http://www.samsung.com - * - * EXYNOS4210 - CPU frequency scaling support - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. -*/ - -#include <linux/module.h> -#include <linux/kernel.h> -#include <linux/err.h> -#include <linux/clk.h> -#include <linux/io.h> -#include <linux/slab.h> -#include <linux/cpufreq.h> - -#include <mach/regs-clock.h> - -#include "exynos-cpufreq.h" - -static struct clk *cpu_clk; -static struct clk *moutcore; -static struct clk *mout_mpll; -static struct clk *mout_apll; - -static unsigned int exynos4210_volt_table[] = { - 1250000, 1150000, 1050000, 975000, 950000, -}; - -static struct cpufreq_frequency_table exynos4210_freq_table[] = { - {L0, 1200 * 1000}, - {L1, 1000 * 1000}, - {L2, 800 * 1000}, - {L3, 500 * 1000}, - {L4, 200 * 1000}, - {0, CPUFREQ_TABLE_END}, -}; - -static struct apll_freq apll_freq_4210[] = { - /* - * values: - * freq - * clock divider for CORE, COREM0, COREM1, PERIPH, ATB, PCLK_DBG, APLL, RESERVED - * clock divider for COPY, HPM, RESERVED - * PLL M, P, S - */ - APLL_FREQ(1200, 0, 3, 7, 3, 4, 1, 7, 0, 5, 0, 0, 150, 3, 1), - APLL_FREQ(1000, 0, 3, 7, 3, 4, 1, 7, 0, 4, 0, 0, 250, 6, 1), - APLL_FREQ(800, 0, 3, 7, 3, 3, 1, 7, 0, 3, 0, 0, 200, 6, 1), - APLL_FREQ(500, 0, 3, 7, 3, 3, 1, 7, 0, 3, 0, 0, 250, 6, 2), - APLL_FREQ(200, 0, 1, 3, 1, 3, 1, 0, 0, 3, 0, 0, 200, 6, 3), -}; - -static void exynos4210_set_clkdiv(unsigned int div_index) -{ - unsigned int tmp; - - /* Change Divider - CPU0 */ - - tmp = apll_freq_4210[div_index].clk_div_cpu0; - - __raw_writel(tmp, EXYNOS4_CLKDIV_CPU); - - do { - tmp = __raw_readl(EXYNOS4_CLKDIV_STATCPU); - } while (tmp & 0x1111111); - - /* Change Divider - CPU1 */ - - tmp = apll_freq_4210[div_index].clk_div_cpu1; - - __raw_writel(tmp, EXYNOS4_CLKDIV_CPU1); - - do { - tmp = __raw_readl(EXYNOS4_CLKDIV_STATCPU1); - } while (tmp & 0x11); -} - -static void exynos4210_set_apll(unsigned int index) -{ - unsigned int tmp; - - /* 1. MUX_CORE_SEL = MPLL, ARMCLK uses MPLL for lock time */ - clk_set_parent(moutcore, mout_mpll); - - do { - tmp = (__raw_readl(EXYNOS4_CLKMUX_STATCPU) - >> EXYNOS4_CLKSRC_CPU_MUXCORE_SHIFT); - tmp &= 0x7; - } while (tmp != 0x2); - - /* 2. Set APLL Lock time */ - __raw_writel(EXYNOS4_APLL_LOCKTIME, EXYNOS4_APLL_LOCK); - - /* 3. Change PLL PMS values */ - tmp = __raw_readl(EXYNOS4_APLL_CON0); - tmp &= ~((0x3ff << 16) | (0x3f << 8) | (0x7 << 0)); - tmp |= apll_freq_4210[index].mps; - __raw_writel(tmp, EXYNOS4_APLL_CON0); - - /* 4. wait_lock_time */ - do { - tmp = __raw_readl(EXYNOS4_APLL_CON0); - } while (!(tmp & (0x1 << EXYNOS4_APLLCON0_LOCKED_SHIFT))); - - /* 5. MUX_CORE_SEL = APLL */ - clk_set_parent(moutcore, mout_apll); - - do { - tmp = __raw_readl(EXYNOS4_CLKMUX_STATCPU); - tmp &= EXYNOS4_CLKMUX_STATCPU_MUXCORE_MASK; - } while (tmp != (0x1 << EXYNOS4_CLKSRC_CPU_MUXCORE_SHIFT)); -} - -static bool exynos4210_pms_change(unsigned int old_index, unsigned int new_index) -{ - unsigned int old_pm = apll_freq_4210[old_index].mps >> 8; - unsigned int new_pm = apll_freq_4210[new_index].mps >> 8; - - return (old_pm == new_pm) ? 0 : 1; -} - -static void exynos4210_set_frequency(unsigned int old_index, - unsigned int new_index) -{ - unsigned int tmp; - - if (old_index > new_index) { - if (!exynos4210_pms_change(old_index, new_index)) { - /* 1. Change the system clock divider values */ - exynos4210_set_clkdiv(new_index); - - /* 2. Change just s value in apll m,p,s value */ - tmp = __raw_readl(EXYNOS4_APLL_CON0); - tmp &= ~(0x7 << 0); - tmp |= apll_freq_4210[new_index].mps & 0x7; - __raw_writel(tmp, EXYNOS4_APLL_CON0); - } else { - /* Clock Configuration Procedure */ - /* 1. Change the system clock divider values */ - exynos4210_set_clkdiv(new_index); - /* 2. Change the apll m,p,s value */ - exynos4210_set_apll(new_index); - } - } else if (old_index < new_index) { - if (!exynos4210_pms_change(old_index, new_index)) { - /* 1. Change just s value in apll m,p,s value */ - tmp = __raw_readl(EXYNOS4_APLL_CON0); - tmp &= ~(0x7 << 0); - tmp |= apll_freq_4210[new_index].mps & 0x7; - __raw_writel(tmp, EXYNOS4_APLL_CON0); - - /* 2. Change the system clock divider values */ - exynos4210_set_clkdiv(new_index); - } else { - /* Clock Configuration Procedure */ - /* 1. Change the apll m,p,s value */ - exynos4210_set_apll(new_index); - /* 2. Change the system clock divider values */ - exynos4210_set_clkdiv(new_index); - } - } -} - -int exynos4210_cpufreq_init(struct exynos_dvfs_info *info) -{ - unsigned long rate; - - cpu_clk = clk_get(NULL, "armclk"); - if (IS_ERR(cpu_clk)) - return PTR_ERR(cpu_clk); - - moutcore = clk_get(NULL, "moutcore"); - if (IS_ERR(moutcore)) - goto err_moutcore; - - mout_mpll = clk_get(NULL, "mout_mpll"); - if (IS_ERR(mout_mpll)) - goto err_mout_mpll; - - rate = clk_get_rate(mout_mpll) / 1000; - - mout_apll = clk_get(NULL, "mout_apll"); - if (IS_ERR(mout_apll)) - goto err_mout_apll; - - info->mpll_freq_khz = rate; - /* 800Mhz */ - info->pll_safe_idx = L2; - info->cpu_clk = cpu_clk; - info->volt_table = exynos4210_volt_table; - info->freq_table = exynos4210_freq_table; - info->set_freq = exynos4210_set_frequency; - info->need_apll_change = exynos4210_pms_change; - - return 0; - -err_mout_apll: - clk_put(mout_mpll); -err_mout_mpll: - clk_put(moutcore); -err_moutcore: - clk_put(cpu_clk); - - pr_debug("%s: failed initialization\n", __func__); - return -EINVAL; -} -EXPORT_SYMBOL(exynos4210_cpufreq_init); diff --git a/drivers/cpufreq/exynos4x12-cpufreq.c b/drivers/cpufreq/exynos4x12-cpufreq.c deleted file mode 100644 index 08b7477b0aa2..000000000000 --- a/drivers/cpufreq/exynos4x12-cpufreq.c +++ /dev/null @@ -1,267 +0,0 @@ -/* - * Copyright (c) 2010-2012 Samsung Electronics Co., Ltd. - * http://www.samsung.com - * - * EXYNOS4X12 - CPU frequency scaling support - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. -*/ - -#include <linux/module.h> -#include <linux/kernel.h> -#include <linux/err.h> -#include <linux/clk.h> -#include <linux/io.h> -#include <linux/slab.h> -#include <linux/cpufreq.h> - -#include <mach/regs-clock.h> - -#include "exynos-cpufreq.h" - -static struct clk *cpu_clk; -static struct clk *moutcore; -static struct clk *mout_mpll; -static struct clk *mout_apll; - -static unsigned int exynos4x12_volt_table[] = { - 1350000, 1287500, 1250000, 1187500, 1137500, 1087500, 1037500, - 1000000, 987500, 975000, 950000, 925000, 900000, 900000 -}; - -static struct cpufreq_frequency_table exynos4x12_freq_table[] = { - {L0, CPUFREQ_ENTRY_INVALID}, - {L1, 1400 * 1000}, - {L2, 1300 * 1000}, - {L3, 1200 * 1000}, - {L4, 1100 * 1000}, - {L5, 1000 * 1000}, - {L6, 900 * 1000}, - {L7, 800 * 1000}, - {L8, 700 * 1000}, - {L9, 600 * 1000}, - {L10, 500 * 1000}, - {L11, 400 * 1000}, - {L12, 300 * 1000}, - {L13, 200 * 1000}, - {0, CPUFREQ_TABLE_END}, -}; - -static struct apll_freq *apll_freq_4x12; - -static struct apll_freq apll_freq_4212[] = { - /* - * values: - * freq - * clock divider for CORE, COREM0, COREM1, PERIPH, ATB, PCLK_DBG, APLL, CORE2 - * clock divider for COPY, HPM, RESERVED - * PLL M, P, S - */ - APLL_FREQ(1500, 0, 3, 7, 0, 6, 1, 2, 0, 6, 2, 0, 250, 4, 0), - APLL_FREQ(1400, 0, 3, 7, 0, 6, 1, 2, 0, 6, 2, 0, 175, 3, 0), - APLL_FREQ(1300, 0, 3, 7, 0, 5, 1, 2, 0, 5, 2, 0, 325, 6, 0), - APLL_FREQ(1200, 0, 3, 7, 0, 5, 1, 2, 0, 5, 2, 0, 200, 4, 0), - APLL_FREQ(1100, 0, 3, 6, 0, 4, 1, 2, 0, 4, 2, 0, 275, 6, 0), - APLL_FREQ(1000, 0, 2, 5, 0, 4, 1, 1, 0, 4, 2, 0, 125, 3, 0), - APLL_FREQ(900, 0, 2, 5, 0, 3, 1, 1, 0, 3, 2, 0, 150, 4, 0), - APLL_FREQ(800, 0, 2, 5, 0, 3, 1, 1, 0, 3, 2, 0, 100, 3, 0), - APLL_FREQ(700, 0, 2, 4, 0, 3, 1, 1, 0, 3, 2, 0, 175, 3, 1), - APLL_FREQ(600, 0, 2, 4, 0, 3, 1, 1, 0, 3, 2, 0, 200, 4, 1), - APLL_FREQ(500, 0, 2, 4, 0, 3, 1, 1, 0, 3, 2, 0, 125, 3, 1), - APLL_FREQ(400, 0, 2, 4, 0, 3, 1, 1, 0, 3, 2, 0, 100, 3, 1), - APLL_FREQ(300, 0, 2, 4, 0, 2, 1, 1, 0, 3, 2, 0, 200, 4, 2), - APLL_FREQ(200, 0, 1, 3, 0, 1, 1, 1, 0, 3, 2, 0, 100, 3, 2), -}; - -static struct apll_freq apll_freq_4412[] = { - /* - * values: - * freq - * clock divider for CORE, COREM0, COREM1, PERIPH, ATB, PCLK_DBG, APLL, CORE2 - * clock divider for COPY, HPM, CORES - * PLL M, P, S - */ - APLL_FREQ(1500, 0, 3, 7, 0, 6, 1, 2, 0, 6, 0, 7, 250, 4, 0), - APLL_FREQ(1400, 0, 3, 7, 0, 6, 1, 2, 0, 6, 0, 6, 175, 3, 0), - APLL_FREQ(1300, 0, 3, 7, 0, 5, 1, 2, 0, 5, 0, 6, 325, 6, 0), - APLL_FREQ(1200, 0, 3, 7, 0, 5, 1, 2, 0, 5, 0, 5, 200, 4, 0), - APLL_FREQ(1100, 0, 3, 6, 0, 4, 1, 2, 0, 4, 0, 5, 275, 6, 0), - APLL_FREQ(1000, 0, 2, 5, 0, 4, 1, 1, 0, 4, 0, 4, 125, 3, 0), - APLL_FREQ(900, 0, 2, 5, 0, 3, 1, 1, 0, 3, 0, 4, 150, 4, 0), - APLL_FREQ(800, 0, 2, 5, 0, 3, 1, 1, 0, 3, 0, 3, 100, 3, 0), - APLL_FREQ(700, 0, 2, 4, 0, 3, 1, 1, 0, 3, 0, 3, 175, 3, 1), - APLL_FREQ(600, 0, 2, 4, 0, 3, 1, 1, 0, 3, 0, 2, 200, 4, 1), - APLL_FREQ(500, 0, 2, 4, 0, 3, 1, 1, 0, 3, 0, 2, 125, 3, 1), - APLL_FREQ(400, 0, 2, 4, 0, 3, 1, 1, 0, 3, 0, 1, 100, 3, 1), - APLL_FREQ(300, 0, 2, 4, 0, 2, 1, 1, 0, 3, 0, 1, 200, 4, 2), - APLL_FREQ(200, 0, 1, 3, 0, 1, 1, 1, 0, 3, 0, 0, 100, 3, 2), -}; - -static void exynos4x12_set_clkdiv(unsigned int div_index) -{ - unsigned int tmp; - unsigned int stat_cpu1; - - /* Change Divider - CPU0 */ - - tmp = apll_freq_4x12[div_index].clk_div_cpu0; - - __raw_writel(tmp, EXYNOS4_CLKDIV_CPU); - - while (__raw_readl(EXYNOS4_CLKDIV_STATCPU) & 0x11111111) - cpu_relax(); - - /* Change Divider - CPU1 */ - tmp = apll_freq_4x12[div_index].clk_div_cpu1; - - __raw_writel(tmp, EXYNOS4_CLKDIV_CPU1); - if (soc_is_exynos4212()) - stat_cpu1 = 0x11; - else - stat_cpu1 = 0x111; - - while (__raw_readl(EXYNOS4_CLKDIV_STATCPU1) & stat_cpu1) - cpu_relax(); -} - -static void exynos4x12_set_apll(unsigned int index) -{ - unsigned int tmp, pdiv; - - /* 1. MUX_CORE_SEL = MPLL, ARMCLK uses MPLL for lock time */ - clk_set_parent(moutcore, mout_mpll); - - do { - cpu_relax(); - tmp = (__raw_readl(EXYNOS4_CLKMUX_STATCPU) - >> EXYNOS4_CLKSRC_CPU_MUXCORE_SHIFT); - tmp &= 0x7; - } while (tmp != 0x2); - - /* 2. Set APLL Lock time */ - pdiv = ((apll_freq_4x12[index].mps >> 8) & 0x3f); - - __raw_writel((pdiv * 250), EXYNOS4_APLL_LOCK); - - /* 3. Change PLL PMS values */ - tmp = __raw_readl(EXYNOS4_APLL_CON0); - tmp &= ~((0x3ff << 16) | (0x3f << 8) | (0x7 << 0)); - tmp |= apll_freq_4x12[index].mps; - __raw_writel(tmp, EXYNOS4_APLL_CON0); - - /* 4. wait_lock_time */ - do { - cpu_relax(); - tmp = __raw_readl(EXYNOS4_APLL_CON0); - } while (!(tmp & (0x1 << EXYNOS4_APLLCON0_LOCKED_SHIFT))); - - /* 5. MUX_CORE_SEL = APLL */ - clk_set_parent(moutcore, mout_apll); - - do { - cpu_relax(); - tmp = __raw_readl(EXYNOS4_CLKMUX_STATCPU); - tmp &= EXYNOS4_CLKMUX_STATCPU_MUXCORE_MASK; - } while (tmp != (0x1 << EXYNOS4_CLKSRC_CPU_MUXCORE_SHIFT)); -} - -static bool exynos4x12_pms_change(unsigned int old_index, unsigned int new_index) -{ - unsigned int old_pm = apll_freq_4x12[old_index].mps >> 8; - unsigned int new_pm = apll_freq_4x12[new_index].mps >> 8; - - return (old_pm == new_pm) ? 0 : 1; -} - -static void exynos4x12_set_frequency(unsigned int old_index, - unsigned int new_index) -{ - unsigned int tmp; - - if (old_index > new_index) { - if (!exynos4x12_pms_change(old_index, new_index)) { - /* 1. Change the system clock divider values */ - exynos4x12_set_clkdiv(new_index); - /* 2. Change just s value in apll m,p,s value */ - tmp = __raw_readl(EXYNOS4_APLL_CON0); - tmp &= ~(0x7 << 0); - tmp |= apll_freq_4x12[new_index].mps & 0x7; - __raw_writel(tmp, EXYNOS4_APLL_CON0); - - } else { - /* Clock Configuration Procedure */ - /* 1. Change the system clock divider values */ - exynos4x12_set_clkdiv(new_index); - /* 2. Change the apll m,p,s value */ - exynos4x12_set_apll(new_index); - } - } else if (old_index < new_index) { - if (!exynos4x12_pms_change(old_index, new_index)) { - /* 1. Change just s value in apll m,p,s value */ - tmp = __raw_readl(EXYNOS4_APLL_CON0); - tmp &= ~(0x7 << 0); - tmp |= apll_freq_4x12[new_index].mps & 0x7; - __raw_writel(tmp, EXYNOS4_APLL_CON0); - /* 2. Change the system clock divider values */ - exynos4x12_set_clkdiv(new_index); - } else { - /* Clock Configuration Procedure */ - /* 1. Change the apll m,p,s value */ - exynos4x12_set_apll(new_index); - /* 2. Change the system clock divider values */ - exynos4x12_set_clkdiv(new_index); - } - } -} - -int exynos4x12_cpufreq_init(struct exynos_dvfs_info *info) -{ - unsigned long rate; - - cpu_clk = clk_get(NULL, "armclk"); - if (IS_ERR(cpu_clk)) - return PTR_ERR(cpu_clk); - - moutcore = clk_get(NULL, "moutcore"); - if (IS_ERR(moutcore)) - goto err_moutcore; - - mout_mpll = clk_get(NULL, "mout_mpll"); - if (IS_ERR(mout_mpll)) - goto err_mout_mpll; - - rate = clk_get_rate(mout_mpll) / 1000; - - mout_apll = clk_get(NULL, "mout_apll"); - if (IS_ERR(mout_apll)) - goto err_mout_apll; - - if (soc_is_exynos4212()) - apll_freq_4x12 = apll_freq_4212; - else - apll_freq_4x12 = apll_freq_4412; - - info->mpll_freq_khz = rate; - /* 800Mhz */ - info->pll_safe_idx = L7; - info->cpu_clk = cpu_clk; - info->volt_table = exynos4x12_volt_table; - info->freq_table = exynos4x12_freq_table; - info->set_freq = exynos4x12_set_frequency; - info->need_apll_change = exynos4x12_pms_change; - - return 0; - -err_mout_apll: - clk_put(mout_mpll); -err_mout_mpll: - clk_put(moutcore); -err_moutcore: - clk_put(cpu_clk); - - pr_debug("%s: failed initialization\n", __func__); - return -EINVAL; -} -EXPORT_SYMBOL(exynos4x12_cpufreq_init); diff --git a/drivers/cpufreq/exynos5250-cpufreq.c b/drivers/cpufreq/exynos5250-cpufreq.c deleted file mode 100644 index 9fae466d7746..000000000000 --- a/drivers/cpufreq/exynos5250-cpufreq.c +++ /dev/null @@ -1,239 +0,0 @@ -/* - * Copyright (c) 2010-20122Samsung Electronics Co., Ltd. - * http://www.samsung.com - * - * EXYNOS5250 - CPU frequency scaling support - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. -*/ - -#include <linux/module.h> -#include <linux/kernel.h> -#include <linux/err.h> -#include <linux/clk.h> -#include <linux/io.h> -#include <linux/slab.h> -#include <linux/cpufreq.h> - -#include <mach/map.h> -#include <mach/regs-clock.h> - -#include "exynos-cpufreq.h" - -static struct clk *cpu_clk; -static struct clk *moutcore; -static struct clk *mout_mpll; -static struct clk *mout_apll; - -static unsigned int exynos5250_volt_table[] = { - 1300000, 1250000, 1225000, 1200000, 1150000, - 1125000, 1100000, 1075000, 1050000, 1025000, - 1012500, 1000000, 975000, 950000, 937500, - 925000 -}; - -static struct cpufreq_frequency_table exynos5250_freq_table[] = { - {L0, 1700 * 1000}, - {L1, 1600 * 1000}, - {L2, 1500 * 1000}, - {L3, 1400 * 1000}, - {L4, 1300 * 1000}, - {L5, 1200 * 1000}, - {L6, 1100 * 1000}, - {L7, 1000 * 1000}, - {L8, 900 * 1000}, - {L9, 800 * 1000}, - {L10, 700 * 1000}, - {L11, 600 * 1000}, - {L12, 500 * 1000}, - {L13, 400 * 1000}, - {L14, 300 * 1000}, - {L15, 200 * 1000}, - {0, CPUFREQ_TABLE_END}, -}; - -static struct apll_freq apll_freq_5250[] = { - /* - * values: - * freq - * clock divider for ARM, CPUD, ACP, PERIPH, ATB, PCLK_DBG, APLL, ARM2 - * clock divider for COPY, HPM, RESERVED - * PLL M, P, S - */ - APLL_FREQ(1700, 0, 3, 7, 7, 7, 3, 5, 0, 0, 2, 0, 425, 6, 0), - APLL_FREQ(1600, 0, 3, 7, 7, 7, 1, 4, 0, 0, 2, 0, 200, 3, 0), - APLL_FREQ(1500, 0, 2, 7, 7, 7, 1, 4, 0, 0, 2, 0, 250, 4, 0), - APLL_FREQ(1400, 0, 2, 7, 7, 6, 1, 4, 0, 0, 2, 0, 175, 3, 0), - APLL_FREQ(1300, 0, 2, 7, 7, 6, 1, 3, 0, 0, 2, 0, 325, 6, 0), - APLL_FREQ(1200, 0, 2, 7, 7, 5, 1, 3, 0, 0, 2, 0, 200, 4, 0), - APLL_FREQ(1100, 0, 3, 7, 7, 5, 1, 3, 0, 0, 2, 0, 275, 6, 0), - APLL_FREQ(1000, 0, 1, 7, 7, 4, 1, 2, 0, 0, 2, 0, 125, 3, 0), - APLL_FREQ(900, 0, 1, 7, 7, 4, 1, 2, 0, 0, 2, 0, 150, 4, 0), - APLL_FREQ(800, 0, 1, 7, 7, 4, 1, 2, 0, 0, 2, 0, 100, 3, 0), - APLL_FREQ(700, 0, 1, 7, 7, 3, 1, 1, 0, 0, 2, 0, 175, 3, 1), - APLL_FREQ(600, 0, 1, 7, 7, 3, 1, 1, 0, 0, 2, 0, 200, 4, 1), - APLL_FREQ(500, 0, 1, 7, 7, 2, 1, 1, 0, 0, 2, 0, 125, 3, 1), - APLL_FREQ(400, 0, 1, 7, 7, 2, 1, 1, 0, 0, 2, 0, 100, 3, 1), - APLL_FREQ(300, 0, 1, 7, 7, 1, 1, 1, 0, 0, 2, 0, 200, 4, 2), - APLL_FREQ(200, 0, 1, 7, 7, 1, 1, 1, 0, 0, 2, 0, 100, 3, 2), -}; - -static void set_clkdiv(unsigned int div_index) -{ - unsigned int tmp; - - /* Change Divider - CPU0 */ - - tmp = apll_freq_5250[div_index].clk_div_cpu0; - - __raw_writel(tmp, EXYNOS5_CLKDIV_CPU0); - - while (__raw_readl(EXYNOS5_CLKDIV_STATCPU0) & 0x11111111) - cpu_relax(); - - /* Change Divider - CPU1 */ - tmp = apll_freq_5250[div_index].clk_div_cpu1; - - __raw_writel(tmp, EXYNOS5_CLKDIV_CPU1); - - while (__raw_readl(EXYNOS5_CLKDIV_STATCPU1) & 0x11) - cpu_relax(); -} - -static void set_apll(unsigned int new_index, - unsigned int old_index) -{ - unsigned int tmp, pdiv; - - /* 1. MUX_CORE_SEL = MPLL, ARMCLK uses MPLL for lock time */ - clk_set_parent(moutcore, mout_mpll); - - do { - cpu_relax(); - tmp = (__raw_readl(EXYNOS5_CLKMUX_STATCPU) >> 16); - tmp &= 0x7; - } while (tmp != 0x2); - - /* 2. Set APLL Lock time */ - pdiv = ((apll_freq_5250[new_index].mps >> 8) & 0x3f); - - __raw_writel((pdiv * 250), EXYNOS5_APLL_LOCK); - - /* 3. Change PLL PMS values */ - tmp = __raw_readl(EXYNOS5_APLL_CON0); - tmp &= ~((0x3ff << 16) | (0x3f << 8) | (0x7 << 0)); - tmp |= apll_freq_5250[new_index].mps; - __raw_writel(tmp, EXYNOS5_APLL_CON0); - - /* 4. wait_lock_time */ - do { - cpu_relax(); - tmp = __raw_readl(EXYNOS5_APLL_CON0); - } while (!(tmp & (0x1 << 29))); - - /* 5. MUX_CORE_SEL = APLL */ - clk_set_parent(moutcore, mout_apll); - - do { - cpu_relax(); - tmp = __raw_readl(EXYNOS5_CLKMUX_STATCPU); - tmp &= (0x7 << 16); - } while (tmp != (0x1 << 16)); - -} - -static bool exynos5250_pms_change(unsigned int old_index, unsigned int new_index) -{ - unsigned int old_pm = apll_freq_5250[old_index].mps >> 8; - unsigned int new_pm = apll_freq_5250[new_index].mps >> 8; - - return (old_pm == new_pm) ? 0 : 1; -} - -static void exynos5250_set_frequency(unsigned int old_index, - unsigned int new_index) -{ - unsigned int tmp; - - if (old_index > new_index) { - if (!exynos5250_pms_change(old_index, new_index)) { - /* 1. Change the system clock divider values */ - set_clkdiv(new_index); - /* 2. Change just s value in apll m,p,s value */ - tmp = __raw_readl(EXYNOS5_APLL_CON0); - tmp &= ~(0x7 << 0); - tmp |= apll_freq_5250[new_index].mps & 0x7; - __raw_writel(tmp, EXYNOS5_APLL_CON0); - - } else { - /* Clock Configuration Procedure */ - /* 1. Change the system clock divider values */ - set_clkdiv(new_index); - /* 2. Change the apll m,p,s value */ - set_apll(new_index, old_index); - } - } else if (old_index < new_index) { - if (!exynos5250_pms_change(old_index, new_index)) { - /* 1. Change just s value in apll m,p,s value */ - tmp = __raw_readl(EXYNOS5_APLL_CON0); - tmp &= ~(0x7 << 0); - tmp |= apll_freq_5250[new_index].mps & 0x7; - __raw_writel(tmp, EXYNOS5_APLL_CON0); - /* 2. Change the system clock divider values */ - set_clkdiv(new_index); - } else { - /* Clock Configuration Procedure */ - /* 1. Change the apll m,p,s value */ - set_apll(new_index, old_index); - /* 2. Change the system clock divider values */ - set_clkdiv(new_index); - } - } -} - -int exynos5250_cpufreq_init(struct exynos_dvfs_info *info) -{ - unsigned long rate; - - cpu_clk = clk_get(NULL, "armclk"); - if (IS_ERR(cpu_clk)) - return PTR_ERR(cpu_clk); - - moutcore = clk_get(NULL, "mout_cpu"); - if (IS_ERR(moutcore)) - goto err_moutcore; - - mout_mpll = clk_get(NULL, "mout_mpll"); - if (IS_ERR(mout_mpll)) - goto err_mout_mpll; - - rate = clk_get_rate(mout_mpll) / 1000; - - mout_apll = clk_get(NULL, "mout_apll"); - if (IS_ERR(mout_apll)) - goto err_mout_apll; - - info->mpll_freq_khz = rate; - /* 800Mhz */ - info->pll_safe_idx = L9; - info->cpu_clk = cpu_clk; - info->volt_table = exynos5250_volt_table; - info->freq_table = exynos5250_freq_table; - info->set_freq = exynos5250_set_frequency; - info->need_apll_change = exynos5250_pms_change; - - return 0; - -err_mout_apll: - clk_put(mout_mpll); -err_mout_mpll: - clk_put(moutcore); -err_moutcore: - clk_put(cpu_clk); - - pr_err("%s: failed initialization\n", __func__); - return -EINVAL; -} -EXPORT_SYMBOL(exynos5250_cpufreq_init); diff --git a/drivers/cpufreq/exynos5440-cpufreq.c b/drivers/cpufreq/exynos5440-cpufreq.c deleted file mode 100644 index 0c74018eda47..000000000000 --- a/drivers/cpufreq/exynos5440-cpufreq.c +++ /dev/null @@ -1,481 +0,0 @@ -/* - * Copyright (c) 2013 Samsung Electronics Co., Ltd. - * http://www.samsung.com - * - * Amit Daniel Kachhap <amit.daniel@samsung.com> - * - * EXYNOS5440 - CPU frequency scaling support - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. -*/ - -#include <linux/clk.h> -#include <linux/cpu.h> -#include <linux/cpufreq.h> -#include <linux/err.h> -#include <linux/interrupt.h> -#include <linux/io.h> -#include <linux/module.h> -#include <linux/of_address.h> -#include <linux/of_irq.h> -#include <linux/opp.h> -#include <linux/platform_device.h> -#include <linux/slab.h> - -/* Register definitions */ -#define XMU_DVFS_CTRL 0x0060 -#define XMU_PMU_P0_7 0x0064 -#define XMU_C0_3_PSTATE 0x0090 -#define XMU_P_LIMIT 0x00a0 -#define XMU_P_STATUS 0x00a4 -#define XMU_PMUEVTEN 0x00d0 -#define XMU_PMUIRQEN 0x00d4 -#define XMU_PMUIRQ 0x00d8 - -/* PMU mask and shift definations */ -#define P_VALUE_MASK 0x7 - -#define XMU_DVFS_CTRL_EN_SHIFT 0 - -#define P0_7_CPUCLKDEV_SHIFT 21 -#define P0_7_CPUCLKDEV_MASK 0x7 -#define P0_7_ATBCLKDEV_SHIFT 18 -#define P0_7_ATBCLKDEV_MASK 0x7 -#define P0_7_CSCLKDEV_SHIFT 15 -#define P0_7_CSCLKDEV_MASK 0x7 -#define P0_7_CPUEMA_SHIFT 28 -#define P0_7_CPUEMA_MASK 0xf -#define P0_7_L2EMA_SHIFT 24 -#define P0_7_L2EMA_MASK 0xf -#define P0_7_VDD_SHIFT 8 -#define P0_7_VDD_MASK 0x7f -#define P0_7_FREQ_SHIFT 0 -#define P0_7_FREQ_MASK 0xff - -#define C0_3_PSTATE_VALID_SHIFT 8 -#define C0_3_PSTATE_CURR_SHIFT 4 -#define C0_3_PSTATE_NEW_SHIFT 0 - -#define PSTATE_CHANGED_EVTEN_SHIFT 0 - -#define PSTATE_CHANGED_IRQEN_SHIFT 0 - -#define PSTATE_CHANGED_SHIFT 0 - -/* some constant values for clock divider calculation */ -#define CPU_DIV_FREQ_MAX 500 -#define CPU_DBG_FREQ_MAX 375 -#define CPU_ATB_FREQ_MAX 500 - -#define PMIC_LOW_VOLT 0x30 -#define PMIC_HIGH_VOLT 0x28 - -#define CPUEMA_HIGH 0x2 -#define CPUEMA_MID 0x4 -#define CPUEMA_LOW 0x7 - -#define L2EMA_HIGH 0x1 -#define L2EMA_MID 0x3 -#define L2EMA_LOW 0x4 - -#define DIV_TAB_MAX 2 -/* frequency unit is 20MHZ */ -#define FREQ_UNIT 20 -#define MAX_VOLTAGE 1550000 /* In microvolt */ -#define VOLTAGE_STEP 12500 /* In microvolt */ - -#define CPUFREQ_NAME "exynos5440_dvfs" -#define DEF_TRANS_LATENCY 100000 - -enum cpufreq_level_index { - L0, L1, L2, L3, L4, - L5, L6, L7, L8, L9, -}; -#define CPUFREQ_LEVEL_END (L7 + 1) - -struct exynos_dvfs_data { - void __iomem *base; - struct resource *mem; - int irq; - struct clk *cpu_clk; - unsigned int cur_frequency; - unsigned int latency; - struct cpufreq_frequency_table *freq_table; - unsigned int freq_count; - struct device *dev; - bool dvfs_enabled; - struct work_struct irq_work; -}; - -static struct exynos_dvfs_data *dvfs_info; -static DEFINE_MUTEX(cpufreq_lock); -static struct cpufreq_freqs freqs; - -static int init_div_table(void) -{ - struct cpufreq_frequency_table *freq_tbl = dvfs_info->freq_table; - unsigned int tmp, clk_div, ema_div, freq, volt_id; - int i = 0; - struct opp *opp; - - rcu_read_lock(); - for (i = 0; freq_tbl[i].frequency != CPUFREQ_TABLE_END; i++) { - - opp = opp_find_freq_exact(dvfs_info->dev, - freq_tbl[i].frequency * 1000, true); - if (IS_ERR(opp)) { - rcu_read_unlock(); - dev_err(dvfs_info->dev, - "failed to find valid OPP for %u KHZ\n", - freq_tbl[i].frequency); - return PTR_ERR(opp); - } - - freq = freq_tbl[i].frequency / 1000; /* In MHZ */ - clk_div = ((freq / CPU_DIV_FREQ_MAX) & P0_7_CPUCLKDEV_MASK) - << P0_7_CPUCLKDEV_SHIFT; - clk_div |= ((freq / CPU_ATB_FREQ_MAX) & P0_7_ATBCLKDEV_MASK) - << P0_7_ATBCLKDEV_SHIFT; - clk_div |= ((freq / CPU_DBG_FREQ_MAX) & P0_7_CSCLKDEV_MASK) - << P0_7_CSCLKDEV_SHIFT; - - /* Calculate EMA */ - volt_id = opp_get_voltage(opp); - volt_id = (MAX_VOLTAGE - volt_id) / VOLTAGE_STEP; - if (volt_id < PMIC_HIGH_VOLT) { - ema_div = (CPUEMA_HIGH << P0_7_CPUEMA_SHIFT) | - (L2EMA_HIGH << P0_7_L2EMA_SHIFT); - } else if (volt_id > PMIC_LOW_VOLT) { - ema_div = (CPUEMA_LOW << P0_7_CPUEMA_SHIFT) | - (L2EMA_LOW << P0_7_L2EMA_SHIFT); - } else { - ema_div = (CPUEMA_MID << P0_7_CPUEMA_SHIFT) | - (L2EMA_MID << P0_7_L2EMA_SHIFT); - } - - tmp = (clk_div | ema_div | (volt_id << P0_7_VDD_SHIFT) - | ((freq / FREQ_UNIT) << P0_7_FREQ_SHIFT)); - - __raw_writel(tmp, dvfs_info->base + XMU_PMU_P0_7 + 4 * i); - } - - rcu_read_unlock(); - return 0; -} - -static void exynos_enable_dvfs(void) -{ - unsigned int tmp, i, cpu; - struct cpufreq_frequency_table *freq_table = dvfs_info->freq_table; - /* Disable DVFS */ - __raw_writel(0, dvfs_info->base + XMU_DVFS_CTRL); - - /* Enable PSTATE Change Event */ - tmp = __raw_readl(dvfs_info->base + XMU_PMUEVTEN); - tmp |= (1 << PSTATE_CHANGED_EVTEN_SHIFT); - __raw_writel(tmp, dvfs_info->base + XMU_PMUEVTEN); - - /* Enable PSTATE Change IRQ */ - tmp = __raw_readl(dvfs_info->base + XMU_PMUIRQEN); - tmp |= (1 << PSTATE_CHANGED_IRQEN_SHIFT); - __raw_writel(tmp, dvfs_info->base + XMU_PMUIRQEN); - - /* Set initial performance index */ - for (i = 0; freq_table[i].frequency != CPUFREQ_TABLE_END; i++) - if (freq_table[i].frequency == dvfs_info->cur_frequency) - break; - - if (freq_table[i].frequency == CPUFREQ_TABLE_END) { - dev_crit(dvfs_info->dev, "Boot up frequency not supported\n"); - /* Assign the highest frequency */ - i = 0; - dvfs_info->cur_frequency = freq_table[i].frequency; - } - - dev_info(dvfs_info->dev, "Setting dvfs initial frequency = %uKHZ", - dvfs_info->cur_frequency); - - for (cpu = 0; cpu < CONFIG_NR_CPUS; cpu++) { - tmp = __raw_readl(dvfs_info->base + XMU_C0_3_PSTATE + cpu * 4); - tmp &= ~(P_VALUE_MASK << C0_3_PSTATE_NEW_SHIFT); - tmp |= (i << C0_3_PSTATE_NEW_SHIFT); - __raw_writel(tmp, dvfs_info->base + XMU_C0_3_PSTATE + cpu * 4); - } - - /* Enable DVFS */ - __raw_writel(1 << XMU_DVFS_CTRL_EN_SHIFT, - dvfs_info->base + XMU_DVFS_CTRL); -} - -static int exynos_verify_speed(struct cpufreq_policy *policy) -{ - return cpufreq_frequency_table_verify(policy, - dvfs_info->freq_table); -} - -static unsigned int exynos_getspeed(unsigned int cpu) -{ - return dvfs_info->cur_frequency; -} - -static int exynos_target(struct cpufreq_policy *policy, - unsigned int target_freq, - unsigned int relation) -{ - unsigned int index, tmp; - int ret = 0, i; - struct cpufreq_frequency_table *freq_table = dvfs_info->freq_table; - - mutex_lock(&cpufreq_lock); - - ret = cpufreq_frequency_table_target(policy, freq_table, - target_freq, relation, &index); - if (ret) - goto out; - - freqs.old = dvfs_info->cur_frequency; - freqs.new = freq_table[index].frequency; - - cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE); - - /* Set the target frequency in all C0_3_PSTATE register */ - for_each_cpu(i, policy->cpus) { - tmp = __raw_readl(dvfs_info->base + XMU_C0_3_PSTATE + i * 4); - tmp &= ~(P_VALUE_MASK << C0_3_PSTATE_NEW_SHIFT); - tmp |= (index << C0_3_PSTATE_NEW_SHIFT); - - __raw_writel(tmp, dvfs_info->base + XMU_C0_3_PSTATE + i * 4); - } -out: - mutex_unlock(&cpufreq_lock); - return ret; -} - -static void exynos_cpufreq_work(struct work_struct *work) -{ - unsigned int cur_pstate, index; - struct cpufreq_policy *policy = cpufreq_cpu_get(0); /* boot CPU */ - struct cpufreq_frequency_table *freq_table = dvfs_info->freq_table; - - /* Ensure we can access cpufreq structures */ - if (unlikely(dvfs_info->dvfs_enabled == false)) - goto skip_work; - - mutex_lock(&cpufreq_lock); - freqs.old = dvfs_info->cur_frequency; - - cur_pstate = __raw_readl(dvfs_info->base + XMU_P_STATUS); - if (cur_pstate >> C0_3_PSTATE_VALID_SHIFT & 0x1) - index = (cur_pstate >> C0_3_PSTATE_CURR_SHIFT) & P_VALUE_MASK; - else - index = (cur_pstate >> C0_3_PSTATE_NEW_SHIFT) & P_VALUE_MASK; - - if (likely(index < dvfs_info->freq_count)) { - freqs.new = freq_table[index].frequency; - dvfs_info->cur_frequency = freqs.new; - } else { - dev_crit(dvfs_info->dev, "New frequency out of range\n"); - freqs.new = dvfs_info->cur_frequency; - } - cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); - - cpufreq_cpu_put(policy); - mutex_unlock(&cpufreq_lock); -skip_work: - enable_irq(dvfs_info->irq); -} - -static irqreturn_t exynos_cpufreq_irq(int irq, void *id) -{ - unsigned int tmp; - - tmp = __raw_readl(dvfs_info->base + XMU_PMUIRQ); - if (tmp >> PSTATE_CHANGED_SHIFT & 0x1) { - __raw_writel(tmp, dvfs_info->base + XMU_PMUIRQ); - disable_irq_nosync(irq); - schedule_work(&dvfs_info->irq_work); - } - return IRQ_HANDLED; -} - -static void exynos_sort_descend_freq_table(void) -{ - struct cpufreq_frequency_table *freq_tbl = dvfs_info->freq_table; - int i = 0, index; - unsigned int tmp_freq; - /* - * Exynos5440 clock controller state logic expects the cpufreq table to - * be in descending order. But the OPP library constructs the table in - * ascending order. So to make the table descending we just need to - * swap the i element with the N - i element. - */ - for (i = 0; i < dvfs_info->freq_count / 2; i++) { - index = dvfs_info->freq_count - i - 1; - tmp_freq = freq_tbl[i].frequency; - freq_tbl[i].frequency = freq_tbl[index].frequency; - freq_tbl[index].frequency = tmp_freq; - } -} - -static int exynos_cpufreq_cpu_init(struct cpufreq_policy *policy) -{ - int ret; - - ret = cpufreq_frequency_table_cpuinfo(policy, dvfs_info->freq_table); - if (ret) { - dev_err(dvfs_info->dev, "Invalid frequency table: %d\n", ret); - return ret; - } - - policy->cur = dvfs_info->cur_frequency; - policy->cpuinfo.transition_latency = dvfs_info->latency; - cpumask_setall(policy->cpus); - - cpufreq_frequency_table_get_attr(dvfs_info->freq_table, policy->cpu); - - return 0; -} - -static struct cpufreq_driver exynos_driver = { - .flags = CPUFREQ_STICKY, - .verify = exynos_verify_speed, - .target = exynos_target, - .get = exynos_getspeed, - .init = exynos_cpufreq_cpu_init, - .name = CPUFREQ_NAME, -}; - -static const struct of_device_id exynos_cpufreq_match[] = { - { - .compatible = "samsung,exynos5440-cpufreq", - }, - {}, -}; -MODULE_DEVICE_TABLE(of, exynos_cpufreq_match); - -static int exynos_cpufreq_probe(struct platform_device *pdev) -{ - int ret = -EINVAL; - struct device_node *np; - struct resource res; - - np = pdev->dev.of_node; - if (!np) - return -ENODEV; - - dvfs_info = devm_kzalloc(&pdev->dev, sizeof(*dvfs_info), GFP_KERNEL); - if (!dvfs_info) { - ret = -ENOMEM; - goto err_put_node; - } - - dvfs_info->dev = &pdev->dev; - - ret = of_address_to_resource(np, 0, &res); - if (ret) - goto err_put_node; - - dvfs_info->base = devm_ioremap_resource(dvfs_info->dev, &res); - if (IS_ERR(dvfs_info->base)) { - ret = PTR_ERR(dvfs_info->base); - goto err_put_node; - } - - dvfs_info->irq = irq_of_parse_and_map(np, 0); - if (!dvfs_info->irq) { - dev_err(dvfs_info->dev, "No cpufreq irq found\n"); - ret = -ENODEV; - goto err_put_node; - } - - ret = of_init_opp_table(dvfs_info->dev); - if (ret) { - dev_err(dvfs_info->dev, "failed to init OPP table: %d\n", ret); - goto err_put_node; - } - - ret = opp_init_cpufreq_table(dvfs_info->dev, &dvfs_info->freq_table); - if (ret) { - dev_err(dvfs_info->dev, - "failed to init cpufreq table: %d\n", ret); - goto err_put_node; - } - dvfs_info->freq_count = opp_get_opp_count(dvfs_info->dev); - exynos_sort_descend_freq_table(); - - if (of_property_read_u32(np, "clock-latency", &dvfs_info->latency)) - dvfs_info->latency = DEF_TRANS_LATENCY; - - dvfs_info->cpu_clk = devm_clk_get(dvfs_info->dev, "armclk"); - if (IS_ERR(dvfs_info->cpu_clk)) { - dev_err(dvfs_info->dev, "Failed to get cpu clock\n"); - ret = PTR_ERR(dvfs_info->cpu_clk); - goto err_free_table; - } - - dvfs_info->cur_frequency = clk_get_rate(dvfs_info->cpu_clk); - if (!dvfs_info->cur_frequency) { - dev_err(dvfs_info->dev, "Failed to get clock rate\n"); - ret = -EINVAL; - goto err_free_table; - } - dvfs_info->cur_frequency /= 1000; - - INIT_WORK(&dvfs_info->irq_work, exynos_cpufreq_work); - ret = devm_request_irq(dvfs_info->dev, dvfs_info->irq, - exynos_cpufreq_irq, IRQF_TRIGGER_NONE, - CPUFREQ_NAME, dvfs_info); - if (ret) { - dev_err(dvfs_info->dev, "Failed to register IRQ\n"); - goto err_free_table; - } - - ret = init_div_table(); - if (ret) { - dev_err(dvfs_info->dev, "Failed to initialise div table\n"); - goto err_free_table; - } - - exynos_enable_dvfs(); - ret = cpufreq_register_driver(&exynos_driver); - if (ret) { - dev_err(dvfs_info->dev, - "%s: failed to register cpufreq driver\n", __func__); - goto err_free_table; - } - - of_node_put(np); - dvfs_info->dvfs_enabled = true; - return 0; - -err_free_table: - opp_free_cpufreq_table(dvfs_info->dev, &dvfs_info->freq_table); -err_put_node: - of_node_put(np); - dev_err(dvfs_info->dev, "%s: failed initialization\n", __func__); - return ret; -} - -static int exynos_cpufreq_remove(struct platform_device *pdev) -{ - cpufreq_unregister_driver(&exynos_driver); - opp_free_cpufreq_table(dvfs_info->dev, &dvfs_info->freq_table); - return 0; -} - -static struct platform_driver exynos_cpufreq_platdrv = { - .driver = { - .name = "exynos5440-cpufreq", - .owner = THIS_MODULE, - .of_match_table = exynos_cpufreq_match, - }, - .probe = exynos_cpufreq_probe, - .remove = exynos_cpufreq_remove, -}; -module_platform_driver(exynos_cpufreq_platdrv); - -MODULE_AUTHOR("Amit Daniel Kachhap <amit.daniel@samsung.com>"); -MODULE_DESCRIPTION("Exynos5440 cpufreq driver"); -MODULE_LICENSE("GPL"); diff --git a/drivers/cpufreq/freq_table.c b/drivers/cpufreq/freq_table.c index f0d87412cc91..7f251daf03ce 100644 --- a/drivers/cpufreq/freq_table.c +++ b/drivers/cpufreq/freq_table.c @@ -1,41 +1,48 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * linux/drivers/cpufreq/freq_table.c * * Copyright (C) 2002 - 2003 Dominik Brodowski - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/init.h> #include <linux/cpufreq.h> +#include <linux/module.h> /********************************************************************* * FREQUENCY TABLE HELPERS * *********************************************************************/ -int cpufreq_frequency_table_cpuinfo(struct cpufreq_policy *policy, - struct cpufreq_frequency_table *table) +static bool policy_has_boost_freq(struct cpufreq_policy *policy) +{ + struct cpufreq_frequency_table *pos, *table = policy->freq_table; + + if (!table) + return false; + + cpufreq_for_each_valid_entry(pos, table) + if (pos->flags & CPUFREQ_BOOST_FREQ) + return true; + + return false; +} + +int cpufreq_frequency_table_cpuinfo(struct cpufreq_policy *policy) { + struct cpufreq_frequency_table *pos, *table = policy->freq_table; unsigned int min_freq = ~0; unsigned int max_freq = 0; - unsigned int i; + unsigned int freq, i; - for (i = 0; (table[i].frequency != CPUFREQ_TABLE_END); i++) { - unsigned int freq = table[i].frequency; - if (freq == CPUFREQ_ENTRY_INVALID) { - pr_debug("table entry %u is invalid, skipping\n", i); + cpufreq_for_each_valid_entry_idx(pos, table, i) { + freq = pos->frequency; + if ((!cpufreq_boost_enabled() || !policy->boost_enabled) + && (pos->flags & CPUFREQ_BOOST_FREQ)) continue; - } - pr_debug("table entry %u: %u kHz, %u driver_data\n", - i, freq, table[i].driver_data); + + pr_debug("table entry %u: %u kHz\n", i, freq); if (freq < min_freq) min_freq = freq; if (freq > max_freq) @@ -43,44 +50,47 @@ int cpufreq_frequency_table_cpuinfo(struct cpufreq_policy *policy, } policy->min = policy->cpuinfo.min_freq = min_freq; - policy->max = policy->cpuinfo.max_freq = max_freq; + policy->max = max_freq; + /* + * If the driver has set its own cpuinfo.max_freq above max_freq, leave + * it as is. + */ + if (policy->cpuinfo.max_freq < max_freq) + policy->max = policy->cpuinfo.max_freq = max_freq; if (policy->min == ~0) return -EINVAL; else return 0; } -EXPORT_SYMBOL_GPL(cpufreq_frequency_table_cpuinfo); - -int cpufreq_frequency_table_verify(struct cpufreq_policy *policy, - struct cpufreq_frequency_table *table) +int cpufreq_frequency_table_verify(struct cpufreq_policy_data *policy) { - unsigned int next_larger = ~0; - unsigned int i; - unsigned int count = 0; + struct cpufreq_frequency_table *pos, *table = policy->freq_table; + unsigned int freq, prev_smaller = 0; + bool found = false; pr_debug("request for verification of policy (%u - %u kHz) for cpu %u\n", policy->min, policy->max, policy->cpu); - cpufreq_verify_within_limits(policy, policy->cpuinfo.min_freq, - policy->cpuinfo.max_freq); + cpufreq_verify_within_cpu_limits(policy); - for (i = 0; (table[i].frequency != CPUFREQ_TABLE_END); i++) { - unsigned int freq = table[i].frequency; - if (freq == CPUFREQ_ENTRY_INVALID) - continue; - if ((freq >= policy->min) && (freq <= policy->max)) - count++; - else if ((next_larger > freq) && (freq > policy->max)) - next_larger = freq; - } + cpufreq_for_each_valid_entry(pos, table) { + freq = pos->frequency; - if (!count) - policy->max = next_larger; + if ((freq >= policy->min) && (freq <= policy->max)) { + found = true; + break; + } + + if ((prev_smaller < freq) && (freq <= policy->max)) + prev_smaller = freq; + } - cpufreq_verify_within_limits(policy, policy->cpuinfo.min_freq, - policy->cpuinfo.max_freq); + if (!found) { + policy->max = prev_smaller; + cpufreq_verify_within_cpu_limits(policy); + } pr_debug("verification lead to (%u - %u kHz) for cpu %u\n", policy->min, policy->max, policy->cpu); @@ -89,12 +99,22 @@ int cpufreq_frequency_table_verify(struct cpufreq_policy *policy, } EXPORT_SYMBOL_GPL(cpufreq_frequency_table_verify); +/* + * Generic routine to verify policy & frequency table, requires driver to set + * policy->freq_table prior to it. + */ +int cpufreq_generic_frequency_table_verify(struct cpufreq_policy_data *policy) +{ + if (!policy->freq_table) + return -ENODEV; + + return cpufreq_frequency_table_verify(policy); +} +EXPORT_SYMBOL_GPL(cpufreq_generic_frequency_table_verify); -int cpufreq_frequency_table_target(struct cpufreq_policy *policy, - struct cpufreq_frequency_table *table, - unsigned int target_freq, - unsigned int relation, - unsigned int *index) +int cpufreq_table_index_unsorted(struct cpufreq_policy *policy, + unsigned int target_freq, unsigned int min, + unsigned int max, unsigned int relation) { struct cpufreq_frequency_table optimal = { .driver_data = ~0, @@ -104,7 +124,10 @@ int cpufreq_frequency_table_target(struct cpufreq_policy *policy, .driver_data = ~0, .frequency = 0, }; - unsigned int i; + struct cpufreq_frequency_table *pos; + struct cpufreq_frequency_table *table = policy->freq_table; + unsigned int freq, diff, i; + int index; pr_debug("request for target %u kHz (relation: %u) for cpu %u\n", target_freq, relation, policy->cpu); @@ -114,19 +137,23 @@ int cpufreq_frequency_table_target(struct cpufreq_policy *policy, suboptimal.frequency = ~0; break; case CPUFREQ_RELATION_L: + case CPUFREQ_RELATION_C: optimal.frequency = ~0; break; } - for (i = 0; (table[i].frequency != CPUFREQ_TABLE_END); i++) { - unsigned int freq = table[i].frequency; - if (freq == CPUFREQ_ENTRY_INVALID) - continue; - if ((freq < policy->min) || (freq > policy->max)) + cpufreq_for_each_valid_entry_idx(pos, table, i) { + freq = pos->frequency; + + if (freq < min || freq > max) continue; + if (freq == target_freq) { + optimal.driver_data = i; + break; + } switch (relation) { case CPUFREQ_RELATION_H: - if (freq <= target_freq) { + if (freq < target_freq) { if (freq >= optimal.frequency) { optimal.frequency = freq; optimal.driver_data = i; @@ -139,7 +166,7 @@ int cpufreq_frequency_table_target(struct cpufreq_policy *policy, } break; case CPUFREQ_RELATION_L: - if (freq >= target_freq) { + if (freq > target_freq) { if (freq <= optimal.frequency) { optimal.frequency = freq; optimal.driver_data = i; @@ -151,42 +178,80 @@ int cpufreq_frequency_table_target(struct cpufreq_policy *policy, } } break; + case CPUFREQ_RELATION_C: + diff = abs(freq - target_freq); + if (diff < optimal.frequency || + (diff == optimal.frequency && + freq > table[optimal.driver_data].frequency)) { + optimal.frequency = diff; + optimal.driver_data = i; + } + break; } } if (optimal.driver_data > i) { - if (suboptimal.driver_data > i) - return -EINVAL; - *index = suboptimal.driver_data; + if (suboptimal.driver_data > i) { + WARN(1, "Invalid frequency table: %u\n", policy->cpu); + return 0; + } + + index = suboptimal.driver_data; } else - *index = optimal.driver_data; + index = optimal.driver_data; - pr_debug("target is %u (%u kHz, %u)\n", *index, table[*index].frequency, - table[*index].driver_data); + pr_debug("target index is %u, freq is:%u kHz\n", index, + table[index].frequency); + return index; +} +EXPORT_SYMBOL_GPL(cpufreq_table_index_unsorted); - return 0; +int cpufreq_frequency_table_get_index(struct cpufreq_policy *policy, + unsigned int freq) +{ + struct cpufreq_frequency_table *pos, *table = policy->freq_table; + int idx; + + if (unlikely(!table)) { + pr_debug("%s: Unable to find frequency table\n", __func__); + return -ENOENT; + } + + cpufreq_for_each_valid_entry_idx(pos, table, idx) + if (pos->frequency == freq) + return idx; + + return -EINVAL; } -EXPORT_SYMBOL_GPL(cpufreq_frequency_table_target); +EXPORT_SYMBOL_GPL(cpufreq_frequency_table_get_index); -static DEFINE_PER_CPU(struct cpufreq_frequency_table *, cpufreq_show_table); -/** +/* * show_available_freqs - show available frequencies for the specified CPU */ -static ssize_t show_available_freqs(struct cpufreq_policy *policy, char *buf) +static ssize_t show_available_freqs(struct cpufreq_policy *policy, char *buf, + bool show_boost) { - unsigned int i = 0; - unsigned int cpu = policy->cpu; ssize_t count = 0; - struct cpufreq_frequency_table *table; + struct cpufreq_frequency_table *pos, *table = policy->freq_table; - if (!per_cpu(cpufreq_show_table, cpu)) + if (!table) return -ENODEV; - table = per_cpu(cpufreq_show_table, cpu); - - for (i = 0; (table[i].frequency != CPUFREQ_TABLE_END); i++) { - if (table[i].frequency == CPUFREQ_ENTRY_INVALID) + cpufreq_for_each_valid_entry(pos, table) { + /* + * show_boost = true and driver_data = BOOST freq + * display BOOST freqs + * + * show_boost = false and driver_data = BOOST freq + * show_boost = true and driver_data != BOOST freq + * continue - do not display anything + * + * show_boost = false and driver_data != BOOST freq + * display NON BOOST freqs + */ + if (show_boost ^ (pos->flags & CPUFREQ_BOOST_FREQ)) continue; - count += sprintf(&buf[count], "%d ", table[i].frequency); + + count += sprintf(&buf[count], "%u ", pos->frequency); } count += sprintf(&buf[count], "\n"); @@ -194,48 +259,109 @@ static ssize_t show_available_freqs(struct cpufreq_policy *policy, char *buf) } -struct freq_attr cpufreq_freq_attr_scaling_available_freqs = { - .attr = { .name = "scaling_available_frequencies", - .mode = 0444, - }, - .show = show_available_freqs, -}; -EXPORT_SYMBOL_GPL(cpufreq_freq_attr_scaling_available_freqs); +#define cpufreq_attr_available_freq(_name) \ +struct freq_attr cpufreq_freq_attr_##_name##_freqs = \ +__ATTR_RO(_name##_frequencies) /* - * if you use these, you must assure that the frequency table is valid - * all the time between get_attr and put_attr! + * scaling_available_frequencies_show - show available normal frequencies for + * the specified CPU */ -void cpufreq_frequency_table_get_attr(struct cpufreq_frequency_table *table, - unsigned int cpu) +static ssize_t scaling_available_frequencies_show(struct cpufreq_policy *policy, + char *buf) { - pr_debug("setting show_table for cpu %u to %p\n", cpu, table); - per_cpu(cpufreq_show_table, cpu) = table; + return show_available_freqs(policy, buf, false); } -EXPORT_SYMBOL_GPL(cpufreq_frequency_table_get_attr); +cpufreq_attr_available_freq(scaling_available); -void cpufreq_frequency_table_put_attr(unsigned int cpu) +/* + * scaling_boost_frequencies_show - show available boost frequencies for + * the specified CPU + */ +static ssize_t scaling_boost_frequencies_show(struct cpufreq_policy *policy, + char *buf) { - pr_debug("clearing show_table for cpu %u\n", cpu); - per_cpu(cpufreq_show_table, cpu) = NULL; + return show_available_freqs(policy, buf, true); } -EXPORT_SYMBOL_GPL(cpufreq_frequency_table_put_attr); +cpufreq_attr_available_freq(scaling_boost); -void cpufreq_frequency_table_update_policy_cpu(struct cpufreq_policy *policy) +static int set_freq_table_sorted(struct cpufreq_policy *policy) { - pr_debug("Updating show_table for new_cpu %u from last_cpu %u\n", - policy->cpu, policy->last_cpu); - per_cpu(cpufreq_show_table, policy->cpu) = per_cpu(cpufreq_show_table, - policy->last_cpu); - per_cpu(cpufreq_show_table, policy->last_cpu) = NULL; + struct cpufreq_frequency_table *pos, *table = policy->freq_table; + struct cpufreq_frequency_table *prev = NULL; + int ascending = 0; + + policy->freq_table_sorted = CPUFREQ_TABLE_UNSORTED; + + cpufreq_for_each_valid_entry(pos, table) { + if (!prev) { + prev = pos; + continue; + } + + if (pos->frequency == prev->frequency) { + pr_warn("Duplicate freq-table entries: %u\n", + pos->frequency); + return -EINVAL; + } + + /* Frequency increased from prev to pos */ + if (pos->frequency > prev->frequency) { + /* But frequency was decreasing earlier */ + if (ascending < 0) { + pr_debug("Freq table is unsorted\n"); + return 0; + } + + ascending++; + } else { + /* Frequency decreased from prev to pos */ + + /* But frequency was increasing earlier */ + if (ascending > 0) { + pr_debug("Freq table is unsorted\n"); + return 0; + } + + ascending--; + } + + prev = pos; + } + + if (ascending > 0) + policy->freq_table_sorted = CPUFREQ_TABLE_SORTED_ASCENDING; + else + policy->freq_table_sorted = CPUFREQ_TABLE_SORTED_DESCENDING; + + pr_debug("Freq table is sorted in %s order\n", + ascending > 0 ? "ascending" : "descending"); + + return 0; } -struct cpufreq_frequency_table *cpufreq_frequency_get_table(unsigned int cpu) +int cpufreq_table_validate_and_sort(struct cpufreq_policy *policy) { - return per_cpu(cpufreq_show_table, cpu); + int ret; + + if (!policy->freq_table) { + /* Freq table must be passed by drivers with target_index() */ + if (has_target_index()) + return -EINVAL; + + return 0; + } + + ret = cpufreq_frequency_table_cpuinfo(policy); + if (ret) + return ret; + + /* Driver's may have set this field already */ + if (policy_has_boost_freq(policy)) + policy->boost_supported = true; + + return set_freq_table_sorted(policy); } -EXPORT_SYMBOL_GPL(cpufreq_frequency_get_table); MODULE_AUTHOR("Dominik Brodowski <linux@brodo.de>"); MODULE_DESCRIPTION("CPUfreq frequency table helpers"); -MODULE_LICENSE("GPL"); diff --git a/drivers/cpufreq/gx-suspmod.c b/drivers/cpufreq/gx-suspmod.c index 3dfc99b9ca86..75b3ef7ec679 100644 --- a/drivers/cpufreq/gx-suspmod.c +++ b/drivers/cpufreq/gx-suspmod.c @@ -1,13 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Cyrix MediaGX and NatSemi Geode Suspend Modulation * (C) 2002 Zwane Mwaikambo <zwane@commfireservices.com> * (C) 2002 Hiroshi Miura <miura@da-cha.org> * All Rights Reserved * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * version 2 as published by the Free Software Foundation - * * The author(s) of this software shall not be held liable for damages * of any nature resulting due to the use of this software. This * software is provided AS-IS with no warranties. @@ -48,7 +45,6 @@ * off_duration = (freq * DURATION) / stock_freq * on_duration = DURATION - off_duration * - * *--------------------------------------------------------------------------- * * ChangeLog: @@ -144,7 +140,7 @@ module_param(max_duration, int, 0444); /** - * we can detect a core multipiler from dir0_lsb + * we can detect a core multiplier from dir0_lsb * from GX1 datasheet p.56, * MULT[3:0]: * 0000 = SYSCLK multiplied by 4 (test only) @@ -183,7 +179,7 @@ static void gx_write_byte(int reg, int value) * gx_detect_chipset: * **/ -static __init struct pci_dev *gx_detect_chipset(void) +static struct pci_dev * __init gx_detect_chipset(void) { struct pci_dev *gx_pci = NULL; @@ -265,7 +261,7 @@ static void gx_set_cpuspeed(struct cpufreq_policy *policy, unsigned int khz) freqs.new = new_khz; - cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE); + cpufreq_freq_transition_begin(policy, &freqs); local_irq_save(flags); if (new_khz != stock_freq) { @@ -314,7 +310,7 @@ static void gx_set_cpuspeed(struct cpufreq_policy *policy, unsigned int khz) gx_params->pci_suscfg = suscfg; - cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); + cpufreq_freq_transition_end(policy, &freqs, 0); pr_debug("suspend modulation w/ duration of ON:%d us, OFF:%d us\n", gx_params->on_duration * 32, gx_params->off_duration * 32); @@ -332,7 +328,7 @@ static void gx_set_cpuspeed(struct cpufreq_policy *policy, unsigned int khz) * for the hardware supported by the driver. */ -static int cpufreq_gx_verify(struct cpufreq_policy *policy) +static int cpufreq_gx_verify(struct cpufreq_policy_data *policy) { unsigned int tmp_freq = 0; u8 tmp1, tmp2; @@ -346,7 +342,7 @@ static int cpufreq_gx_verify(struct cpufreq_policy *policy) /* it needs to be assured that at least one supported frequency is * within policy->min and policy->max. If it is not, policy->max - * needs to be increased until one freuqency is supported. + * needs to be increased until one frequency is supported. * policy->min may not be decreased, though. This way we guarantee a * specific processing capacity. */ @@ -401,7 +397,7 @@ static int cpufreq_gx_target(struct cpufreq_policy *policy, static int cpufreq_gx_cpu_init(struct cpufreq_policy *policy) { - unsigned int maxfreq, curfreq; + unsigned int maxfreq; if (!policy || policy->cpu != 0) return -ENODEV; @@ -415,10 +411,8 @@ static int cpufreq_gx_cpu_init(struct cpufreq_policy *policy) maxfreq = 30000 * gx_freq_mult[getCx86(CX86_DIR1) & 0x0f]; stock_freq = maxfreq; - curfreq = gx_get_cpuspeed(0); pr_debug("cpu max frequency is %d.\n", maxfreq); - pr_debug("cpu current frequency is %dkHz.\n", curfreq); /* setup basic struct for cpufreq API */ policy->cpu = 0; @@ -428,10 +422,8 @@ static int cpufreq_gx_cpu_init(struct cpufreq_policy *policy) else policy->min = maxfreq / POLICY_MIN_DIV; policy->max = maxfreq; - policy->cur = curfreq; policy->cpuinfo.min_freq = maxfreq / max_duration; policy->cpuinfo.max_freq = maxfreq; - policy->cpuinfo.transition_latency = CPUFREQ_ETERNAL; return 0; } @@ -441,12 +433,12 @@ static int cpufreq_gx_cpu_init(struct cpufreq_policy *policy) * MediaGX/Geode GX initialize cpufreq driver */ static struct cpufreq_driver gx_suspmod_driver = { + .flags = CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING, .get = gx_get_cpuspeed, .verify = cpufreq_gx_verify, .target = cpufreq_gx_target, .init = cpufreq_gx_cpu_init, .name = "gx-suspmod", - .owner = THIS_MODULE, }; static int __init cpufreq_gx_init(void) @@ -466,7 +458,7 @@ static int __init cpufreq_gx_init(void) pr_debug("geode suspend modulation available.\n"); - params = kzalloc(sizeof(struct gxfreq_params), GFP_KERNEL); + params = kzalloc(sizeof(*params), GFP_KERNEL); if (params == NULL) return -ENOMEM; diff --git a/drivers/cpufreq/highbank-cpufreq.c b/drivers/cpufreq/highbank-cpufreq.c index b61b5a3fad64..a45864701143 100644 --- a/drivers/cpufreq/highbank-cpufreq.c +++ b/drivers/cpufreq/highbank-cpufreq.c @@ -1,12 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2012 Calxeda, Inc. * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * * This driver provides the clk notifier callbacks that are used when - * the cpufreq-cpu0 driver changes to frequency to alert the highbank + * the cpufreq-dt driver changes to frequency to alert the highbank * EnergyCore Management Engine (ECME) about the need to change * voltage. The ECME interfaces with the actual voltage regulators. */ @@ -19,7 +16,7 @@ #include <linux/cpu.h> #include <linux/err.h> #include <linux/of.h> -#include <linux/mailbox.h> +#include <linux/pl320-ipc.h> #include <linux/platform_device.h> #define HB_CPUFREQ_CHANGE_NOTE 0x80000001 @@ -58,34 +55,29 @@ static struct notifier_block hb_cpufreq_clk_nb = { .notifier_call = hb_cpufreq_clk_notify, }; -static int hb_cpufreq_driver_init(void) +static int __init hb_cpufreq_driver_init(void) { - struct platform_device_info devinfo = { .name = "cpufreq-cpu0", }; + struct platform_device_info devinfo = { .name = "cpufreq-dt", }; struct device *cpu_dev; struct clk *cpu_clk; struct device_node *np; int ret; - if (!of_machine_is_compatible("calxeda,highbank")) + if ((!of_machine_is_compatible("calxeda,highbank")) && + (!of_machine_is_compatible("calxeda,ecx-2000"))) return -ENODEV; - for_each_child_of_node(of_find_node_by_path("/cpus"), np) - if (of_get_property(np, "operating-points", NULL)) - break; - - if (!np) { - pr_err("failed to find highbank cpufreq node\n"); - return -ENOENT; - } - cpu_dev = get_cpu_device(0); if (!cpu_dev) { pr_err("failed to get highbank cpufreq device\n"); - ret = -ENODEV; - goto out_put_node; + return -ENODEV; } - cpu_dev->of_node = np; + np = of_node_get(cpu_dev->of_node); + if (!np) { + pr_err("failed to find highbank cpufreq node\n"); + return -ENOENT; + } cpu_clk = clk_get(cpu_dev, NULL); if (IS_ERR(cpu_clk)) { @@ -100,7 +92,7 @@ static int hb_cpufreq_driver_init(void) goto out_put_node; } - /* Instantiate cpufreq-cpu0 */ + /* Instantiate cpufreq-dt */ platform_device_register_full(&devinfo); out_put_node: @@ -109,6 +101,13 @@ out_put_node: } module_init(hb_cpufreq_driver_init); +static const struct of_device_id __maybe_unused hb_cpufreq_of_match[] = { + { .compatible = "calxeda,highbank" }, + { .compatible = "calxeda,ecx-2000" }, + { }, +}; +MODULE_DEVICE_TABLE(of, hb_cpufreq_of_match); + MODULE_AUTHOR("Mark Langsdorf <mark.langsdorf@calxeda.com>"); MODULE_DESCRIPTION("Calxeda Highbank cpufreq driver"); MODULE_LICENSE("GPL"); diff --git a/drivers/cpufreq/ia64-acpi-cpufreq.c b/drivers/cpufreq/ia64-acpi-cpufreq.c deleted file mode 100644 index 573c14ea802d..000000000000 --- a/drivers/cpufreq/ia64-acpi-cpufreq.c +++ /dev/null @@ -1,438 +0,0 @@ -/* - * This file provides the ACPI based P-state support. This - * module works with generic cpufreq infrastructure. Most of - * the code is based on i386 version - * (arch/i386/kernel/cpu/cpufreq/acpi-cpufreq.c) - * - * Copyright (C) 2005 Intel Corp - * Venkatesh Pallipadi <venkatesh.pallipadi@intel.com> - */ - -#include <linux/kernel.h> -#include <linux/slab.h> -#include <linux/module.h> -#include <linux/init.h> -#include <linux/cpufreq.h> -#include <linux/proc_fs.h> -#include <linux/seq_file.h> -#include <asm/io.h> -#include <asm/uaccess.h> -#include <asm/pal.h> - -#include <linux/acpi.h> -#include <acpi/processor.h> - -MODULE_AUTHOR("Venkatesh Pallipadi"); -MODULE_DESCRIPTION("ACPI Processor P-States Driver"); -MODULE_LICENSE("GPL"); - - -struct cpufreq_acpi_io { - struct acpi_processor_performance acpi_data; - struct cpufreq_frequency_table *freq_table; - unsigned int resume; -}; - -static struct cpufreq_acpi_io *acpi_io_data[NR_CPUS]; - -static struct cpufreq_driver acpi_cpufreq_driver; - - -static int -processor_set_pstate ( - u32 value) -{ - s64 retval; - - pr_debug("processor_set_pstate\n"); - - retval = ia64_pal_set_pstate((u64)value); - - if (retval) { - pr_debug("Failed to set freq to 0x%x, with error 0x%lx\n", - value, retval); - return -ENODEV; - } - return (int)retval; -} - - -static int -processor_get_pstate ( - u32 *value) -{ - u64 pstate_index = 0; - s64 retval; - - pr_debug("processor_get_pstate\n"); - - retval = ia64_pal_get_pstate(&pstate_index, - PAL_GET_PSTATE_TYPE_INSTANT); - *value = (u32) pstate_index; - - if (retval) - pr_debug("Failed to get current freq with " - "error 0x%lx, idx 0x%x\n", retval, *value); - - return (int)retval; -} - - -/* To be used only after data->acpi_data is initialized */ -static unsigned -extract_clock ( - struct cpufreq_acpi_io *data, - unsigned value, - unsigned int cpu) -{ - unsigned long i; - - pr_debug("extract_clock\n"); - - for (i = 0; i < data->acpi_data.state_count; i++) { - if (value == data->acpi_data.states[i].status) - return data->acpi_data.states[i].core_frequency; - } - return data->acpi_data.states[i-1].core_frequency; -} - - -static unsigned int -processor_get_freq ( - struct cpufreq_acpi_io *data, - unsigned int cpu) -{ - int ret = 0; - u32 value = 0; - cpumask_t saved_mask; - unsigned long clock_freq; - - pr_debug("processor_get_freq\n"); - - saved_mask = current->cpus_allowed; - set_cpus_allowed_ptr(current, cpumask_of(cpu)); - if (smp_processor_id() != cpu) - goto migrate_end; - - /* processor_get_pstate gets the instantaneous frequency */ - ret = processor_get_pstate(&value); - - if (ret) { - set_cpus_allowed_ptr(current, &saved_mask); - printk(KERN_WARNING "get performance failed with error %d\n", - ret); - ret = 0; - goto migrate_end; - } - clock_freq = extract_clock(data, value, cpu); - ret = (clock_freq*1000); - -migrate_end: - set_cpus_allowed_ptr(current, &saved_mask); - return ret; -} - - -static int -processor_set_freq ( - struct cpufreq_acpi_io *data, - struct cpufreq_policy *policy, - int state) -{ - int ret = 0; - u32 value = 0; - struct cpufreq_freqs cpufreq_freqs; - cpumask_t saved_mask; - int retval; - - pr_debug("processor_set_freq\n"); - - saved_mask = current->cpus_allowed; - set_cpus_allowed_ptr(current, cpumask_of(policy->cpu)); - if (smp_processor_id() != policy->cpu) { - retval = -EAGAIN; - goto migrate_end; - } - - if (state == data->acpi_data.state) { - if (unlikely(data->resume)) { - pr_debug("Called after resume, resetting to P%d\n", state); - data->resume = 0; - } else { - pr_debug("Already at target state (P%d)\n", state); - retval = 0; - goto migrate_end; - } - } - - pr_debug("Transitioning from P%d to P%d\n", - data->acpi_data.state, state); - - /* cpufreq frequency struct */ - cpufreq_freqs.old = data->freq_table[data->acpi_data.state].frequency; - cpufreq_freqs.new = data->freq_table[state].frequency; - - /* notify cpufreq */ - cpufreq_notify_transition(policy, &cpufreq_freqs, CPUFREQ_PRECHANGE); - - /* - * First we write the target state's 'control' value to the - * control_register. - */ - - value = (u32) data->acpi_data.states[state].control; - - pr_debug("Transitioning to state: 0x%08x\n", value); - - ret = processor_set_pstate(value); - if (ret) { - unsigned int tmp = cpufreq_freqs.new; - cpufreq_notify_transition(policy, &cpufreq_freqs, - CPUFREQ_POSTCHANGE); - cpufreq_freqs.new = cpufreq_freqs.old; - cpufreq_freqs.old = tmp; - cpufreq_notify_transition(policy, &cpufreq_freqs, - CPUFREQ_PRECHANGE); - cpufreq_notify_transition(policy, &cpufreq_freqs, - CPUFREQ_POSTCHANGE); - printk(KERN_WARNING "Transition failed with error %d\n", ret); - retval = -ENODEV; - goto migrate_end; - } - - cpufreq_notify_transition(policy, &cpufreq_freqs, CPUFREQ_POSTCHANGE); - - data->acpi_data.state = state; - - retval = 0; - -migrate_end: - set_cpus_allowed_ptr(current, &saved_mask); - return (retval); -} - - -static unsigned int -acpi_cpufreq_get ( - unsigned int cpu) -{ - struct cpufreq_acpi_io *data = acpi_io_data[cpu]; - - pr_debug("acpi_cpufreq_get\n"); - - return processor_get_freq(data, cpu); -} - - -static int -acpi_cpufreq_target ( - struct cpufreq_policy *policy, - unsigned int target_freq, - unsigned int relation) -{ - struct cpufreq_acpi_io *data = acpi_io_data[policy->cpu]; - unsigned int next_state = 0; - unsigned int result = 0; - - pr_debug("acpi_cpufreq_setpolicy\n"); - - result = cpufreq_frequency_table_target(policy, - data->freq_table, target_freq, relation, &next_state); - if (result) - return (result); - - result = processor_set_freq(data, policy, next_state); - - return (result); -} - - -static int -acpi_cpufreq_verify ( - struct cpufreq_policy *policy) -{ - unsigned int result = 0; - struct cpufreq_acpi_io *data = acpi_io_data[policy->cpu]; - - pr_debug("acpi_cpufreq_verify\n"); - - result = cpufreq_frequency_table_verify(policy, - data->freq_table); - - return (result); -} - - -static int -acpi_cpufreq_cpu_init ( - struct cpufreq_policy *policy) -{ - unsigned int i; - unsigned int cpu = policy->cpu; - struct cpufreq_acpi_io *data; - unsigned int result = 0; - - pr_debug("acpi_cpufreq_cpu_init\n"); - - data = kzalloc(sizeof(struct cpufreq_acpi_io), GFP_KERNEL); - if (!data) - return (-ENOMEM); - - acpi_io_data[cpu] = data; - - result = acpi_processor_register_performance(&data->acpi_data, cpu); - - if (result) - goto err_free; - - /* capability check */ - if (data->acpi_data.state_count <= 1) { - pr_debug("No P-States\n"); - result = -ENODEV; - goto err_unreg; - } - - if ((data->acpi_data.control_register.space_id != - ACPI_ADR_SPACE_FIXED_HARDWARE) || - (data->acpi_data.status_register.space_id != - ACPI_ADR_SPACE_FIXED_HARDWARE)) { - pr_debug("Unsupported address space [%d, %d]\n", - (u32) (data->acpi_data.control_register.space_id), - (u32) (data->acpi_data.status_register.space_id)); - result = -ENODEV; - goto err_unreg; - } - - /* alloc freq_table */ - data->freq_table = kmalloc(sizeof(struct cpufreq_frequency_table) * - (data->acpi_data.state_count + 1), - GFP_KERNEL); - if (!data->freq_table) { - result = -ENOMEM; - goto err_unreg; - } - - /* detect transition latency */ - policy->cpuinfo.transition_latency = 0; - for (i=0; i<data->acpi_data.state_count; i++) { - if ((data->acpi_data.states[i].transition_latency * 1000) > - policy->cpuinfo.transition_latency) { - policy->cpuinfo.transition_latency = - data->acpi_data.states[i].transition_latency * 1000; - } - } - policy->cur = processor_get_freq(data, policy->cpu); - - /* table init */ - for (i = 0; i <= data->acpi_data.state_count; i++) - { - data->freq_table[i].driver_data = i; - if (i < data->acpi_data.state_count) { - data->freq_table[i].frequency = - data->acpi_data.states[i].core_frequency * 1000; - } else { - data->freq_table[i].frequency = CPUFREQ_TABLE_END; - } - } - - result = cpufreq_frequency_table_cpuinfo(policy, data->freq_table); - if (result) { - goto err_freqfree; - } - - /* notify BIOS that we exist */ - acpi_processor_notify_smm(THIS_MODULE); - - printk(KERN_INFO "acpi-cpufreq: CPU%u - ACPI performance management " - "activated.\n", cpu); - - for (i = 0; i < data->acpi_data.state_count; i++) - pr_debug(" %cP%d: %d MHz, %d mW, %d uS, %d uS, 0x%x 0x%x\n", - (i == data->acpi_data.state?'*':' '), i, - (u32) data->acpi_data.states[i].core_frequency, - (u32) data->acpi_data.states[i].power, - (u32) data->acpi_data.states[i].transition_latency, - (u32) data->acpi_data.states[i].bus_master_latency, - (u32) data->acpi_data.states[i].status, - (u32) data->acpi_data.states[i].control); - - cpufreq_frequency_table_get_attr(data->freq_table, policy->cpu); - - /* the first call to ->target() should result in us actually - * writing something to the appropriate registers. */ - data->resume = 1; - - return (result); - - err_freqfree: - kfree(data->freq_table); - err_unreg: - acpi_processor_unregister_performance(&data->acpi_data, cpu); - err_free: - kfree(data); - acpi_io_data[cpu] = NULL; - - return (result); -} - - -static int -acpi_cpufreq_cpu_exit ( - struct cpufreq_policy *policy) -{ - struct cpufreq_acpi_io *data = acpi_io_data[policy->cpu]; - - pr_debug("acpi_cpufreq_cpu_exit\n"); - - if (data) { - cpufreq_frequency_table_put_attr(policy->cpu); - acpi_io_data[policy->cpu] = NULL; - acpi_processor_unregister_performance(&data->acpi_data, - policy->cpu); - kfree(data); - } - - return (0); -} - - -static struct freq_attr* acpi_cpufreq_attr[] = { - &cpufreq_freq_attr_scaling_available_freqs, - NULL, -}; - - -static struct cpufreq_driver acpi_cpufreq_driver = { - .verify = acpi_cpufreq_verify, - .target = acpi_cpufreq_target, - .get = acpi_cpufreq_get, - .init = acpi_cpufreq_cpu_init, - .exit = acpi_cpufreq_cpu_exit, - .name = "acpi-cpufreq", - .owner = THIS_MODULE, - .attr = acpi_cpufreq_attr, -}; - - -static int __init -acpi_cpufreq_init (void) -{ - pr_debug("acpi_cpufreq_init\n"); - - return cpufreq_register_driver(&acpi_cpufreq_driver); -} - - -static void __exit -acpi_cpufreq_exit (void) -{ - pr_debug("acpi_cpufreq_exit\n"); - - cpufreq_unregister_driver(&acpi_cpufreq_driver); - return; -} - - -late_initcall(acpi_cpufreq_init); -module_exit(acpi_cpufreq_exit); - diff --git a/drivers/cpufreq/imx-cpufreq-dt.c b/drivers/cpufreq/imx-cpufreq-dt.c new file mode 100644 index 000000000000..1492c92ffc1a --- /dev/null +++ b/drivers/cpufreq/imx-cpufreq-dt.c @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2019 NXP + */ + +#include <linux/clk.h> +#include <linux/cpu.h> +#include <linux/cpufreq.h> +#include <linux/err.h> +#include <linux/init.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/nvmem-consumer.h> +#include <linux/of.h> +#include <linux/platform_device.h> +#include <linux/pm_opp.h> +#include <linux/regulator/consumer.h> +#include <linux/slab.h> + +#include "cpufreq-dt.h" + +#define OCOTP_CFG3_SPEED_GRADE_SHIFT 8 +#define OCOTP_CFG3_SPEED_GRADE_MASK (0x3 << 8) +#define IMX8MN_OCOTP_CFG3_SPEED_GRADE_MASK (0xf << 8) +#define OCOTP_CFG3_MKT_SEGMENT_SHIFT 6 +#define OCOTP_CFG3_MKT_SEGMENT_MASK (0x3 << 6) +#define IMX8MP_OCOTP_CFG3_MKT_SEGMENT_SHIFT 5 +#define IMX8MP_OCOTP_CFG3_MKT_SEGMENT_MASK (0x3 << 5) + +#define IMX7ULP_MAX_RUN_FREQ 528000 + +/* cpufreq-dt device registered by imx-cpufreq-dt */ +static struct platform_device *cpufreq_dt_pdev; +static struct device *cpu_dev; +static int cpufreq_opp_token; + +enum IMX7ULP_CPUFREQ_CLKS { + ARM, + CORE, + SCS_SEL, + HSRUN_CORE, + HSRUN_SCS_SEL, + FIRC, +}; + +static struct clk_bulk_data imx7ulp_clks[] = { + { .id = "arm" }, + { .id = "core" }, + { .id = "scs_sel" }, + { .id = "hsrun_core" }, + { .id = "hsrun_scs_sel" }, + { .id = "firc" }, +}; + +static unsigned int imx7ulp_get_intermediate(struct cpufreq_policy *policy, + unsigned int index) +{ + return clk_get_rate(imx7ulp_clks[FIRC].clk); +} + +static int imx7ulp_target_intermediate(struct cpufreq_policy *policy, + unsigned int index) +{ + unsigned int newfreq = policy->freq_table[index].frequency; + + clk_set_parent(imx7ulp_clks[SCS_SEL].clk, imx7ulp_clks[FIRC].clk); + clk_set_parent(imx7ulp_clks[HSRUN_SCS_SEL].clk, imx7ulp_clks[FIRC].clk); + + if (newfreq > IMX7ULP_MAX_RUN_FREQ) + clk_set_parent(imx7ulp_clks[ARM].clk, + imx7ulp_clks[HSRUN_CORE].clk); + else + clk_set_parent(imx7ulp_clks[ARM].clk, imx7ulp_clks[CORE].clk); + + return 0; +} + +static struct cpufreq_dt_platform_data imx7ulp_data = { + .target_intermediate = imx7ulp_target_intermediate, + .get_intermediate = imx7ulp_get_intermediate, +}; + +static int imx_cpufreq_dt_probe(struct platform_device *pdev) +{ + struct platform_device *dt_pdev; + u32 cell_value, supported_hw[2]; + int speed_grade, mkt_segment; + int ret; + + cpu_dev = get_cpu_device(0); + + if (!of_property_present(cpu_dev->of_node, "cpu-supply")) + return -ENODEV; + + if (of_machine_is_compatible("fsl,imx7ulp")) { + ret = clk_bulk_get(cpu_dev, ARRAY_SIZE(imx7ulp_clks), + imx7ulp_clks); + if (ret) + return ret; + + dt_pdev = platform_device_register_data(NULL, "cpufreq-dt", + -1, &imx7ulp_data, + sizeof(imx7ulp_data)); + if (IS_ERR(dt_pdev)) { + clk_bulk_put(ARRAY_SIZE(imx7ulp_clks), imx7ulp_clks); + ret = PTR_ERR(dt_pdev); + dev_err(&pdev->dev, "Failed to register cpufreq-dt: %d\n", ret); + return ret; + } + + cpufreq_dt_pdev = dt_pdev; + + return 0; + } + + ret = nvmem_cell_read_u32(cpu_dev, "speed_grade", &cell_value); + if (ret) + return ret; + + if (of_machine_is_compatible("fsl,imx8mn") || + of_machine_is_compatible("fsl,imx8mp")) + speed_grade = (cell_value & IMX8MN_OCOTP_CFG3_SPEED_GRADE_MASK) + >> OCOTP_CFG3_SPEED_GRADE_SHIFT; + else + speed_grade = (cell_value & OCOTP_CFG3_SPEED_GRADE_MASK) + >> OCOTP_CFG3_SPEED_GRADE_SHIFT; + + if (of_machine_is_compatible("fsl,imx8mp")) + mkt_segment = (cell_value & IMX8MP_OCOTP_CFG3_MKT_SEGMENT_MASK) + >> IMX8MP_OCOTP_CFG3_MKT_SEGMENT_SHIFT; + else + mkt_segment = (cell_value & OCOTP_CFG3_MKT_SEGMENT_MASK) + >> OCOTP_CFG3_MKT_SEGMENT_SHIFT; + + /* + * Early samples without fuses written report "0 0" which may NOT + * match any OPP defined in DT. So clamp to minimum OPP defined in + * DT to avoid warning for "no OPPs". + * + * Applies to i.MX8M series SoCs. + */ + if (mkt_segment == 0 && speed_grade == 0) { + if (of_machine_is_compatible("fsl,imx8mm") || + of_machine_is_compatible("fsl,imx8mq")) + speed_grade = 1; + if (of_machine_is_compatible("fsl,imx8mn") || + of_machine_is_compatible("fsl,imx8mp")) + speed_grade = 0xb; + } + + supported_hw[0] = BIT(speed_grade); + supported_hw[1] = BIT(mkt_segment); + dev_info(&pdev->dev, "cpu speed grade %d mkt segment %d supported-hw %#x %#x\n", + speed_grade, mkt_segment, supported_hw[0], supported_hw[1]); + + cpufreq_opp_token = dev_pm_opp_set_supported_hw(cpu_dev, supported_hw, 2); + if (cpufreq_opp_token < 0) { + ret = cpufreq_opp_token; + dev_err(&pdev->dev, "Failed to set supported opp: %d\n", ret); + return ret; + } + + cpufreq_dt_pdev = platform_device_register_data( + &pdev->dev, "cpufreq-dt", -1, NULL, 0); + if (IS_ERR(cpufreq_dt_pdev)) { + dev_pm_opp_put_supported_hw(cpufreq_opp_token); + ret = PTR_ERR(cpufreq_dt_pdev); + dev_err(&pdev->dev, "Failed to register cpufreq-dt: %d\n", ret); + return ret; + } + + return 0; +} + +static void imx_cpufreq_dt_remove(struct platform_device *pdev) +{ + platform_device_unregister(cpufreq_dt_pdev); + if (!of_machine_is_compatible("fsl,imx7ulp")) + dev_pm_opp_put_supported_hw(cpufreq_opp_token); + else + clk_bulk_put(ARRAY_SIZE(imx7ulp_clks), imx7ulp_clks); +} + +static struct platform_driver imx_cpufreq_dt_driver = { + .probe = imx_cpufreq_dt_probe, + .remove = imx_cpufreq_dt_remove, + .driver = { + .name = "imx-cpufreq-dt", + }, +}; +module_platform_driver(imx_cpufreq_dt_driver); + +MODULE_ALIAS("platform:imx-cpufreq-dt"); +MODULE_DESCRIPTION("Freescale i.MX cpufreq speed grading driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/cpufreq/imx6q-cpufreq.c b/drivers/cpufreq/imx6q-cpufreq.c index e37cdaedbb5b..e93697d3edfd 100644 --- a/drivers/cpufreq/imx6q-cpufreq.c +++ b/drivers/cpufreq/imx6q-cpufreq.c @@ -1,20 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2013 Freescale Semiconductor, Inc. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #include <linux/clk.h> +#include <linux/cpu.h> #include <linux/cpufreq.h> -#include <linux/delay.h> #include <linux/err.h> #include <linux/module.h> +#include <linux/nvmem-consumer.h> #include <linux/of.h> -#include <linux/opp.h> +#include <linux/of_address.h> +#include <linux/pm_opp.h> #include <linux/platform_device.h> #include <linux/regulator/consumer.h> +#include <linux/mfd/syscon.h> +#include <linux/regmap.h> #define PU_SOC_VOLTAGE_NORMAL 1250000 #define PU_SOC_VOLTAGE_HIGH 1275000 @@ -24,87 +25,84 @@ static struct regulator *arm_reg; static struct regulator *pu_reg; static struct regulator *soc_reg; -static struct clk *arm_clk; -static struct clk *pll1_sys_clk; -static struct clk *pll1_sw_clk; -static struct clk *step_clk; -static struct clk *pll2_pfd2_396m_clk; +enum IMX6_CPUFREQ_CLKS { + ARM, + PLL1_SYS, + STEP, + PLL1_SW, + PLL2_PFD2_396M, + /* MX6UL requires two more clks */ + PLL2_BUS, + SECONDARY_SEL, +}; +#define IMX6Q_CPUFREQ_CLK_NUM 5 +#define IMX6UL_CPUFREQ_CLK_NUM 7 + +static int num_clks; +static struct clk_bulk_data clks[] = { + { .id = "arm" }, + { .id = "pll1_sys" }, + { .id = "step" }, + { .id = "pll1_sw" }, + { .id = "pll2_pfd2_396m" }, + { .id = "pll2_bus" }, + { .id = "secondary_sel" }, +}; static struct device *cpu_dev; static struct cpufreq_frequency_table *freq_table; +static unsigned int max_freq; static unsigned int transition_latency; -static int imx6q_verify_speed(struct cpufreq_policy *policy) -{ - return cpufreq_frequency_table_verify(policy, freq_table); -} - -static unsigned int imx6q_get_speed(unsigned int cpu) -{ - return clk_get_rate(arm_clk) / 1000; -} +static u32 *imx6_soc_volt; +static u32 soc_opp_count; -static int imx6q_set_target(struct cpufreq_policy *policy, - unsigned int target_freq, unsigned int relation) +static int imx6q_set_target(struct cpufreq_policy *policy, unsigned int index) { - struct cpufreq_freqs freqs; - struct opp *opp; + struct dev_pm_opp *opp; unsigned long freq_hz, volt, volt_old; - unsigned int index; + unsigned int old_freq, new_freq; + bool pll1_sys_temp_enabled = false; int ret; - ret = cpufreq_frequency_table_target(policy, freq_table, target_freq, - relation, &index); - if (ret) { - dev_err(cpu_dev, "failed to match target frequency %d: %d\n", - target_freq, ret); - return ret; - } - - freqs.new = freq_table[index].frequency; - freq_hz = freqs.new * 1000; - freqs.old = clk_get_rate(arm_clk) / 1000; - - if (freqs.old == freqs.new) - return 0; + new_freq = freq_table[index].frequency; + freq_hz = new_freq * 1000; + old_freq = clk_get_rate(clks[ARM].clk) / 1000; - rcu_read_lock(); - opp = opp_find_freq_ceil(cpu_dev, &freq_hz); + opp = dev_pm_opp_find_freq_ceil(cpu_dev, &freq_hz); if (IS_ERR(opp)) { - rcu_read_unlock(); dev_err(cpu_dev, "failed to find OPP for %ld\n", freq_hz); return PTR_ERR(opp); } - volt = opp_get_voltage(opp); - rcu_read_unlock(); + volt = dev_pm_opp_get_voltage(opp); + dev_pm_opp_put(opp); + volt_old = regulator_get_voltage(arm_reg); dev_dbg(cpu_dev, "%u MHz, %ld mV --> %u MHz, %ld mV\n", - freqs.old / 1000, volt_old / 1000, - freqs.new / 1000, volt / 1000); - - cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE); + old_freq / 1000, volt_old / 1000, + new_freq / 1000, volt / 1000); /* scaling up? scale voltage before frequency */ - if (freqs.new > freqs.old) { + if (new_freq > old_freq) { + if (!IS_ERR(pu_reg)) { + ret = regulator_set_voltage_tol(pu_reg, imx6_soc_volt[index], 0); + if (ret) { + dev_err(cpu_dev, "failed to scale vddpu up: %d\n", ret); + return ret; + } + } + ret = regulator_set_voltage_tol(soc_reg, imx6_soc_volt[index], 0); + if (ret) { + dev_err(cpu_dev, "failed to scale vddsoc up: %d\n", ret); + return ret; + } ret = regulator_set_voltage_tol(arm_reg, volt, 0); if (ret) { dev_err(cpu_dev, "failed to scale vddarm up: %d\n", ret); - freqs.new = freqs.old; - goto post_notify; - } - - /* - * Need to increase vddpu and vddsoc for safety - * if we are about to run at 1.2 GHz. - */ - if (freqs.new == FREQ_1P2_GHZ / 1000) { - regulator_set_voltage_tol(pu_reg, - PU_SOC_VOLTAGE_HIGH, 0); - regulator_set_voltage_tol(soc_reg, - PU_SOC_VOLTAGE_HIGH, 0); + return ret; } } @@ -112,190 +110,371 @@ static int imx6q_set_target(struct cpufreq_policy *policy, * The setpoints are selected per PLL/PDF frequencies, so we need to * reprogram PLL for frequency scaling. The procedure of reprogramming * PLL1 is as below. - * + * For i.MX6UL, it has a secondary clk mux, the cpu frequency change + * flow is slightly different from other i.MX6 OSC. + * The cpu frequeny change flow for i.MX6(except i.MX6UL) is as below: * - Enable pll2_pfd2_396m_clk and reparent pll1_sw_clk to it * - Reprogram pll1_sys_clk and reparent pll1_sw_clk back to it * - Disable pll2_pfd2_396m_clk */ - clk_prepare_enable(pll2_pfd2_396m_clk); - clk_set_parent(step_clk, pll2_pfd2_396m_clk); - clk_set_parent(pll1_sw_clk, step_clk); - if (freq_hz > clk_get_rate(pll2_pfd2_396m_clk)) { - clk_set_rate(pll1_sys_clk, freqs.new * 1000); + if (of_machine_is_compatible("fsl,imx6ul") || + of_machine_is_compatible("fsl,imx6ull")) { /* - * If we are leaving 396 MHz set-point, we need to enable - * pll1_sys_clk and disable pll2_pfd2_396m_clk to keep - * their use count correct. + * When changing pll1_sw_clk's parent to pll1_sys_clk, + * CPU may run at higher than 528MHz, this will lead to + * the system unstable if the voltage is lower than the + * voltage of 528MHz, so lower the CPU frequency to one + * half before changing CPU frequency. */ - if (freqs.old * 1000 <= clk_get_rate(pll2_pfd2_396m_clk)) { - clk_prepare_enable(pll1_sys_clk); - clk_disable_unprepare(pll2_pfd2_396m_clk); + clk_set_rate(clks[ARM].clk, (old_freq >> 1) * 1000); + clk_set_parent(clks[PLL1_SW].clk, clks[PLL1_SYS].clk); + if (freq_hz > clk_get_rate(clks[PLL2_PFD2_396M].clk)) + clk_set_parent(clks[SECONDARY_SEL].clk, + clks[PLL2_BUS].clk); + else + clk_set_parent(clks[SECONDARY_SEL].clk, + clks[PLL2_PFD2_396M].clk); + clk_set_parent(clks[STEP].clk, clks[SECONDARY_SEL].clk); + clk_set_parent(clks[PLL1_SW].clk, clks[STEP].clk); + if (freq_hz > clk_get_rate(clks[PLL2_BUS].clk)) { + clk_set_rate(clks[PLL1_SYS].clk, new_freq * 1000); + clk_set_parent(clks[PLL1_SW].clk, clks[PLL1_SYS].clk); } - clk_set_parent(pll1_sw_clk, pll1_sys_clk); - clk_disable_unprepare(pll2_pfd2_396m_clk); } else { - /* - * Disable pll1_sys_clk if pll2_pfd2_396m_clk is sufficient - * to provide the frequency. - */ - clk_disable_unprepare(pll1_sys_clk); + clk_set_parent(clks[STEP].clk, clks[PLL2_PFD2_396M].clk); + clk_set_parent(clks[PLL1_SW].clk, clks[STEP].clk); + if (freq_hz > clk_get_rate(clks[PLL2_PFD2_396M].clk)) { + clk_set_rate(clks[PLL1_SYS].clk, new_freq * 1000); + clk_set_parent(clks[PLL1_SW].clk, clks[PLL1_SYS].clk); + } else { + /* pll1_sys needs to be enabled for divider rate change to work. */ + pll1_sys_temp_enabled = true; + clk_prepare_enable(clks[PLL1_SYS].clk); + } } /* Ensure the arm clock divider is what we expect */ - ret = clk_set_rate(arm_clk, freqs.new * 1000); + ret = clk_set_rate(clks[ARM].clk, new_freq * 1000); if (ret) { + int ret1; + dev_err(cpu_dev, "failed to set clock rate: %d\n", ret); - regulator_set_voltage_tol(arm_reg, volt_old, 0); - freqs.new = freqs.old; - goto post_notify; + ret1 = regulator_set_voltage_tol(arm_reg, volt_old, 0); + if (ret1) + dev_warn(cpu_dev, + "failed to restore vddarm voltage: %d\n", ret1); + return ret; } + /* PLL1 is only needed until after ARM-PODF is set. */ + if (pll1_sys_temp_enabled) + clk_disable_unprepare(clks[PLL1_SYS].clk); + /* scaling down? scale voltage after frequency */ - if (freqs.new < freqs.old) { + if (new_freq < old_freq) { ret = regulator_set_voltage_tol(arm_reg, volt, 0); - if (ret) { + if (ret) dev_warn(cpu_dev, "failed to scale vddarm down: %d\n", ret); - ret = 0; - } - - if (freqs.old == FREQ_1P2_GHZ / 1000) { - regulator_set_voltage_tol(pu_reg, - PU_SOC_VOLTAGE_NORMAL, 0); - regulator_set_voltage_tol(soc_reg, - PU_SOC_VOLTAGE_NORMAL, 0); + ret = regulator_set_voltage_tol(soc_reg, imx6_soc_volt[index], 0); + if (ret) + dev_warn(cpu_dev, "failed to scale vddsoc down: %d\n", ret); + if (!IS_ERR(pu_reg)) { + ret = regulator_set_voltage_tol(pu_reg, imx6_soc_volt[index], 0); + if (ret) + dev_warn(cpu_dev, "failed to scale vddpu down: %d\n", ret); } } -post_notify: - cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); - - return ret; + return 0; } static int imx6q_cpufreq_init(struct cpufreq_policy *policy) { + policy->clk = clks[ARM].clk; + cpufreq_generic_init(policy, freq_table, transition_latency); + policy->suspend_freq = max_freq; + + return 0; +} + +static struct cpufreq_driver imx6q_cpufreq_driver = { + .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK | + CPUFREQ_IS_COOLING_DEV, + .verify = cpufreq_generic_frequency_table_verify, + .target_index = imx6q_set_target, + .get = cpufreq_generic_get, + .init = imx6q_cpufreq_init, + .register_em = cpufreq_register_em_with_opp, + .name = "imx6q-cpufreq", + .suspend = cpufreq_generic_suspend, +}; + +static void imx6x_disable_freq_in_opp(struct device *dev, unsigned long freq) +{ + int ret = dev_pm_opp_disable(dev, freq); + + if (ret < 0 && ret != -ENODEV) + dev_warn(dev, "failed to disable %ldMHz OPP\n", freq / 1000000); +} + +#define OCOTP_CFG3 0x440 +#define OCOTP_CFG3_SPEED_SHIFT 16 +#define OCOTP_CFG3_SPEED_1P2GHZ 0x3 +#define OCOTP_CFG3_SPEED_996MHZ 0x2 +#define OCOTP_CFG3_SPEED_852MHZ 0x1 + +static int imx6q_opp_check_speed_grading(struct device *dev) +{ + u32 val; int ret; - ret = cpufreq_frequency_table_cpuinfo(policy, freq_table); - if (ret) { - dev_err(cpu_dev, "invalid frequency table: %d\n", ret); - return ret; + if (of_property_present(dev->of_node, "nvmem-cells")) { + ret = nvmem_cell_read_u32(dev, "speed_grade", &val); + if (ret) + return ret; + } else { + struct regmap *ocotp; + + ocotp = syscon_regmap_lookup_by_compatible("fsl,imx6q-ocotp"); + if (IS_ERR(ocotp)) + return -ENOENT; + + /* + * SPEED_GRADING[1:0] defines the max speed of ARM: + * 2b'11: 1200000000Hz; + * 2b'10: 996000000Hz; + * 2b'01: 852000000Hz; -- i.MX6Q Only, exclusive with 996MHz. + * 2b'00: 792000000Hz; + * We need to set the max speed of ARM according to fuse map. + */ + regmap_read(ocotp, OCOTP_CFG3, &val); } - policy->cpuinfo.transition_latency = transition_latency; - policy->cur = clk_get_rate(arm_clk) / 1000; - cpumask_setall(policy->cpus); - cpufreq_frequency_table_get_attr(freq_table, policy->cpu); + val >>= OCOTP_CFG3_SPEED_SHIFT; + val &= 0x3; + + if (val < OCOTP_CFG3_SPEED_996MHZ) + imx6x_disable_freq_in_opp(dev, 996000000); + + if (of_machine_is_compatible("fsl,imx6q") || + of_machine_is_compatible("fsl,imx6qp")) { + if (val != OCOTP_CFG3_SPEED_852MHZ) + imx6x_disable_freq_in_opp(dev, 852000000); + + if (val != OCOTP_CFG3_SPEED_1P2GHZ) + imx6x_disable_freq_in_opp(dev, 1200000000); + } return 0; } -static int imx6q_cpufreq_exit(struct cpufreq_policy *policy) +#define OCOTP_CFG3_6UL_SPEED_696MHZ 0x2 +#define OCOTP_CFG3_6ULL_SPEED_792MHZ 0x2 +#define OCOTP_CFG3_6ULL_SPEED_900MHZ 0x3 + +static int imx6ul_opp_check_speed_grading(struct device *dev) { - cpufreq_frequency_table_put_attr(policy->cpu); - return 0; -} + u32 val; + int ret = 0; -static struct freq_attr *imx6q_cpufreq_attr[] = { - &cpufreq_freq_attr_scaling_available_freqs, - NULL, -}; + if (of_property_present(dev->of_node, "nvmem-cells")) { + ret = nvmem_cell_read_u32(dev, "speed_grade", &val); + if (ret) + return ret; + } else { + struct regmap *ocotp; -static struct cpufreq_driver imx6q_cpufreq_driver = { - .verify = imx6q_verify_speed, - .target = imx6q_set_target, - .get = imx6q_get_speed, - .init = imx6q_cpufreq_init, - .exit = imx6q_cpufreq_exit, - .name = "imx6q-cpufreq", - .attr = imx6q_cpufreq_attr, -}; + ocotp = syscon_regmap_lookup_by_compatible("fsl,imx6ul-ocotp"); + if (IS_ERR(ocotp)) + ocotp = syscon_regmap_lookup_by_compatible("fsl,imx6ull-ocotp"); + + if (IS_ERR(ocotp)) + return -ENOENT; + + regmap_read(ocotp, OCOTP_CFG3, &val); + } + + /* + * Speed GRADING[1:0] defines the max speed of ARM: + * 2b'00: Reserved; + * 2b'01: 528000000Hz; + * 2b'10: 696000000Hz on i.MX6UL, 792000000Hz on i.MX6ULL; + * 2b'11: 900000000Hz on i.MX6ULL only; + * We need to set the max speed of ARM according to fuse map. + */ + val >>= OCOTP_CFG3_SPEED_SHIFT; + val &= 0x3; + + if (of_machine_is_compatible("fsl,imx6ul")) + if (val != OCOTP_CFG3_6UL_SPEED_696MHZ) + imx6x_disable_freq_in_opp(dev, 696000000); + + if (of_machine_is_compatible("fsl,imx6ull")) { + if (val < OCOTP_CFG3_6ULL_SPEED_792MHZ) + imx6x_disable_freq_in_opp(dev, 792000000); + + if (val != OCOTP_CFG3_6ULL_SPEED_900MHZ) + imx6x_disable_freq_in_opp(dev, 900000000); + } + + return ret; +} static int imx6q_cpufreq_probe(struct platform_device *pdev) { struct device_node *np; - struct opp *opp; + struct dev_pm_opp *opp; unsigned long min_volt, max_volt; int num, ret; + const struct property *prop; + const __be32 *val; + u32 nr, i, j; + + cpu_dev = get_cpu_device(0); + if (!cpu_dev) { + pr_err("failed to get cpu0 device\n"); + return -ENODEV; + } - cpu_dev = &pdev->dev; - - np = of_find_node_by_path("/cpus/cpu@0"); + np = of_node_get(cpu_dev->of_node); if (!np) { dev_err(cpu_dev, "failed to find cpu0 node\n"); return -ENOENT; } - cpu_dev->of_node = np; + if (of_machine_is_compatible("fsl,imx6ul") || + of_machine_is_compatible("fsl,imx6ull")) + num_clks = IMX6UL_CPUFREQ_CLK_NUM; + else + num_clks = IMX6Q_CPUFREQ_CLK_NUM; - arm_clk = devm_clk_get(cpu_dev, "arm"); - pll1_sys_clk = devm_clk_get(cpu_dev, "pll1_sys"); - pll1_sw_clk = devm_clk_get(cpu_dev, "pll1_sw"); - step_clk = devm_clk_get(cpu_dev, "step"); - pll2_pfd2_396m_clk = devm_clk_get(cpu_dev, "pll2_pfd2_396m"); - if (IS_ERR(arm_clk) || IS_ERR(pll1_sys_clk) || IS_ERR(pll1_sw_clk) || - IS_ERR(step_clk) || IS_ERR(pll2_pfd2_396m_clk)) { - dev_err(cpu_dev, "failed to get clocks\n"); - ret = -ENOENT; + ret = clk_bulk_get(cpu_dev, num_clks, clks); + if (ret) goto put_node; - } - arm_reg = devm_regulator_get(cpu_dev, "arm"); - pu_reg = devm_regulator_get(cpu_dev, "pu"); - soc_reg = devm_regulator_get(cpu_dev, "soc"); - if (IS_ERR(arm_reg) || IS_ERR(pu_reg) || IS_ERR(soc_reg)) { + arm_reg = regulator_get(cpu_dev, "arm"); + pu_reg = regulator_get_optional(cpu_dev, "pu"); + soc_reg = regulator_get(cpu_dev, "soc"); + if (PTR_ERR(arm_reg) == -EPROBE_DEFER || + PTR_ERR(soc_reg) == -EPROBE_DEFER || + PTR_ERR(pu_reg) == -EPROBE_DEFER) { + ret = -EPROBE_DEFER; + dev_dbg(cpu_dev, "regulators not ready, defer\n"); + goto put_reg; + } + if (IS_ERR(arm_reg) || IS_ERR(soc_reg)) { dev_err(cpu_dev, "failed to get regulators\n"); ret = -ENOENT; - goto put_node; + goto put_reg; + } + + ret = dev_pm_opp_of_add_table(cpu_dev); + if (ret < 0) { + dev_err(cpu_dev, "failed to init OPP table: %d\n", ret); + goto put_reg; + } + + if (of_machine_is_compatible("fsl,imx6ul") || + of_machine_is_compatible("fsl,imx6ull")) { + ret = imx6ul_opp_check_speed_grading(cpu_dev); + } else { + ret = imx6q_opp_check_speed_grading(cpu_dev); + } + if (ret) { + dev_err_probe(cpu_dev, ret, "failed to read ocotp\n"); + goto out_free_opp; } - /* We expect an OPP table supplied by platform */ - num = opp_get_opp_count(cpu_dev); + num = dev_pm_opp_get_opp_count(cpu_dev); if (num < 0) { ret = num; dev_err(cpu_dev, "no OPP table is found: %d\n", ret); - goto put_node; + goto out_free_opp; } - ret = opp_init_cpufreq_table(cpu_dev, &freq_table); + ret = dev_pm_opp_init_cpufreq_table(cpu_dev, &freq_table); if (ret) { dev_err(cpu_dev, "failed to init cpufreq table: %d\n", ret); - goto put_node; + goto out_free_opp; + } + + /* Make imx6_soc_volt array's size same as arm opp number */ + imx6_soc_volt = devm_kcalloc(cpu_dev, num, sizeof(*imx6_soc_volt), + GFP_KERNEL); + if (imx6_soc_volt == NULL) { + ret = -ENOMEM; + goto free_freq_table; + } + + prop = of_find_property(np, "fsl,soc-operating-points", NULL); + if (!prop || !prop->value) + goto soc_opp_out; + + /* + * Each OPP is a set of tuples consisting of frequency and + * voltage like <freq-kHz vol-uV>. + */ + nr = prop->length / sizeof(u32); + if (nr % 2 || (nr / 2) < num) + goto soc_opp_out; + + for (j = 0; j < num; j++) { + val = prop->value; + for (i = 0; i < nr / 2; i++) { + unsigned long freq = be32_to_cpup(val++); + unsigned long volt = be32_to_cpup(val++); + if (freq_table[j].frequency == freq) { + imx6_soc_volt[soc_opp_count++] = volt; + break; + } + } + } + +soc_opp_out: + /* use fixed soc opp volt if no valid soc opp info found in dtb */ + if (soc_opp_count != num) { + dev_warn(cpu_dev, "can NOT find valid fsl,soc-operating-points property in dtb, use default value!\n"); + for (j = 0; j < num; j++) + imx6_soc_volt[j] = PU_SOC_VOLTAGE_NORMAL; + if (freq_table[num - 1].frequency * 1000 == FREQ_1P2_GHZ) + imx6_soc_volt[num - 1] = PU_SOC_VOLTAGE_HIGH; } if (of_property_read_u32(np, "clock-latency", &transition_latency)) - transition_latency = CPUFREQ_ETERNAL; + transition_latency = CPUFREQ_DEFAULT_TRANSITION_LATENCY_NS; + + /* + * Calculate the ramp time for max voltage change in the + * VDDSOC and VDDPU regulators. + */ + ret = regulator_set_voltage_time(soc_reg, imx6_soc_volt[0], imx6_soc_volt[num - 1]); + if (ret > 0) + transition_latency += ret * 1000; + if (!IS_ERR(pu_reg)) { + ret = regulator_set_voltage_time(pu_reg, imx6_soc_volt[0], imx6_soc_volt[num - 1]); + if (ret > 0) + transition_latency += ret * 1000; + } /* * OPP is maintained in order of increasing frequency, and * freq_table initialised from OPP is therefore sorted in the * same order. */ - rcu_read_lock(); - opp = opp_find_freq_exact(cpu_dev, + max_freq = freq_table[--num].frequency; + opp = dev_pm_opp_find_freq_exact(cpu_dev, freq_table[0].frequency * 1000, true); - min_volt = opp_get_voltage(opp); - opp = opp_find_freq_exact(cpu_dev, - freq_table[--num].frequency * 1000, true); - max_volt = opp_get_voltage(opp); - rcu_read_unlock(); + min_volt = dev_pm_opp_get_voltage(opp); + dev_pm_opp_put(opp); + opp = dev_pm_opp_find_freq_exact(cpu_dev, max_freq * 1000, true); + max_volt = dev_pm_opp_get_voltage(opp); + dev_pm_opp_put(opp); + ret = regulator_set_voltage_time(arm_reg, min_volt, max_volt); if (ret > 0) transition_latency += ret * 1000; - /* Count vddpu and vddsoc latency in for 1.2 GHz support */ - if (freq_table[num].frequency == FREQ_1P2_GHZ / 1000) { - ret = regulator_set_voltage_time(pu_reg, PU_SOC_VOLTAGE_NORMAL, - PU_SOC_VOLTAGE_HIGH); - if (ret > 0) - transition_latency += ret * 1000; - ret = regulator_set_voltage_time(soc_reg, PU_SOC_VOLTAGE_NORMAL, - PU_SOC_VOLTAGE_HIGH); - if (ret > 0) - transition_latency += ret * 1000; - } - ret = cpufreq_register_driver(&imx6q_cpufreq_driver); if (ret) { dev_err(cpu_dev, "failed register driver: %d\n", ret); @@ -306,30 +485,47 @@ static int imx6q_cpufreq_probe(struct platform_device *pdev) return 0; free_freq_table: - opp_free_cpufreq_table(cpu_dev, &freq_table); + dev_pm_opp_free_cpufreq_table(cpu_dev, &freq_table); +out_free_opp: + dev_pm_opp_of_remove_table(cpu_dev); +put_reg: + if (!IS_ERR(arm_reg)) + regulator_put(arm_reg); + if (!IS_ERR(pu_reg)) + regulator_put(pu_reg); + if (!IS_ERR(soc_reg)) + regulator_put(soc_reg); + + clk_bulk_put(num_clks, clks); put_node: of_node_put(np); + return ret; } -static int imx6q_cpufreq_remove(struct platform_device *pdev) +static void imx6q_cpufreq_remove(struct platform_device *pdev) { cpufreq_unregister_driver(&imx6q_cpufreq_driver); - opp_free_cpufreq_table(cpu_dev, &freq_table); - - return 0; + dev_pm_opp_free_cpufreq_table(cpu_dev, &freq_table); + dev_pm_opp_of_remove_table(cpu_dev); + regulator_put(arm_reg); + if (!IS_ERR(pu_reg)) + regulator_put(pu_reg); + regulator_put(soc_reg); + + clk_bulk_put(num_clks, clks); } static struct platform_driver imx6q_cpufreq_platdrv = { .driver = { .name = "imx6q-cpufreq", - .owner = THIS_MODULE, }, .probe = imx6q_cpufreq_probe, .remove = imx6q_cpufreq_remove, }; module_platform_driver(imx6q_cpufreq_platdrv); +MODULE_ALIAS("platform:imx6q-cpufreq"); MODULE_AUTHOR("Shawn Guo <shawn.guo@linaro.org>"); MODULE_DESCRIPTION("Freescale i.MX6Q cpufreq driver"); MODULE_LICENSE("GPL"); diff --git a/drivers/cpufreq/integrator-cpufreq.c b/drivers/cpufreq/integrator-cpufreq.c deleted file mode 100644 index f7c99df0880b..000000000000 --- a/drivers/cpufreq/integrator-cpufreq.c +++ /dev/null @@ -1,220 +0,0 @@ -/* - * Copyright (C) 2001-2002 Deep Blue Solutions Ltd. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * CPU support functions - */ -#include <linux/module.h> -#include <linux/types.h> -#include <linux/kernel.h> -#include <linux/cpufreq.h> -#include <linux/sched.h> -#include <linux/smp.h> -#include <linux/init.h> -#include <linux/io.h> - -#include <mach/hardware.h> -#include <mach/platform.h> -#include <asm/mach-types.h> -#include <asm/hardware/icst.h> - -static struct cpufreq_driver integrator_driver; - -#define CM_ID __io_address(INTEGRATOR_HDR_ID) -#define CM_OSC __io_address(INTEGRATOR_HDR_OSC) -#define CM_STAT __io_address(INTEGRATOR_HDR_STAT) -#define CM_LOCK __io_address(INTEGRATOR_HDR_LOCK) - -static const struct icst_params lclk_params = { - .ref = 24000000, - .vco_max = ICST525_VCO_MAX_5V, - .vco_min = ICST525_VCO_MIN, - .vd_min = 8, - .vd_max = 132, - .rd_min = 24, - .rd_max = 24, - .s2div = icst525_s2div, - .idx2s = icst525_idx2s, -}; - -static const struct icst_params cclk_params = { - .ref = 24000000, - .vco_max = ICST525_VCO_MAX_5V, - .vco_min = ICST525_VCO_MIN, - .vd_min = 12, - .vd_max = 160, - .rd_min = 24, - .rd_max = 24, - .s2div = icst525_s2div, - .idx2s = icst525_idx2s, -}; - -/* - * Validate the speed policy. - */ -static int integrator_verify_policy(struct cpufreq_policy *policy) -{ - struct icst_vco vco; - - cpufreq_verify_within_limits(policy, - policy->cpuinfo.min_freq, - policy->cpuinfo.max_freq); - - vco = icst_hz_to_vco(&cclk_params, policy->max * 1000); - policy->max = icst_hz(&cclk_params, vco) / 1000; - - vco = icst_hz_to_vco(&cclk_params, policy->min * 1000); - policy->min = icst_hz(&cclk_params, vco) / 1000; - - cpufreq_verify_within_limits(policy, - policy->cpuinfo.min_freq, - policy->cpuinfo.max_freq); - - return 0; -} - - -static int integrator_set_target(struct cpufreq_policy *policy, - unsigned int target_freq, - unsigned int relation) -{ - cpumask_t cpus_allowed; - int cpu = policy->cpu; - struct icst_vco vco; - struct cpufreq_freqs freqs; - u_int cm_osc; - - /* - * Save this threads cpus_allowed mask. - */ - cpus_allowed = current->cpus_allowed; - - /* - * Bind to the specified CPU. When this call returns, - * we should be running on the right CPU. - */ - set_cpus_allowed(current, cpumask_of_cpu(cpu)); - BUG_ON(cpu != smp_processor_id()); - - /* get current setting */ - cm_osc = __raw_readl(CM_OSC); - - if (machine_is_integrator()) { - vco.s = (cm_osc >> 8) & 7; - } else if (machine_is_cintegrator()) { - vco.s = 1; - } - vco.v = cm_osc & 255; - vco.r = 22; - freqs.old = icst_hz(&cclk_params, vco) / 1000; - - /* icst_hz_to_vco rounds down -- so we need the next - * larger freq in case of CPUFREQ_RELATION_L. - */ - if (relation == CPUFREQ_RELATION_L) - target_freq += 999; - if (target_freq > policy->max) - target_freq = policy->max; - vco = icst_hz_to_vco(&cclk_params, target_freq * 1000); - freqs.new = icst_hz(&cclk_params, vco) / 1000; - - if (freqs.old == freqs.new) { - set_cpus_allowed(current, cpus_allowed); - return 0; - } - - cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE); - - cm_osc = __raw_readl(CM_OSC); - - if (machine_is_integrator()) { - cm_osc &= 0xfffff800; - cm_osc |= vco.s << 8; - } else if (machine_is_cintegrator()) { - cm_osc &= 0xffffff00; - } - cm_osc |= vco.v; - - __raw_writel(0xa05f, CM_LOCK); - __raw_writel(cm_osc, CM_OSC); - __raw_writel(0, CM_LOCK); - - /* - * Restore the CPUs allowed mask. - */ - set_cpus_allowed(current, cpus_allowed); - - cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); - - return 0; -} - -static unsigned int integrator_get(unsigned int cpu) -{ - cpumask_t cpus_allowed; - unsigned int current_freq; - u_int cm_osc; - struct icst_vco vco; - - cpus_allowed = current->cpus_allowed; - - set_cpus_allowed(current, cpumask_of_cpu(cpu)); - BUG_ON(cpu != smp_processor_id()); - - /* detect memory etc. */ - cm_osc = __raw_readl(CM_OSC); - - if (machine_is_integrator()) { - vco.s = (cm_osc >> 8) & 7; - } else { - vco.s = 1; - } - vco.v = cm_osc & 255; - vco.r = 22; - - current_freq = icst_hz(&cclk_params, vco) / 1000; /* current freq */ - - set_cpus_allowed(current, cpus_allowed); - - return current_freq; -} - -static int integrator_cpufreq_init(struct cpufreq_policy *policy) -{ - - /* set default policy and cpuinfo */ - policy->cpuinfo.max_freq = 160000; - policy->cpuinfo.min_freq = 12000; - policy->cpuinfo.transition_latency = 1000000; /* 1 ms, assumed */ - policy->cur = policy->min = policy->max = integrator_get(policy->cpu); - - return 0; -} - -static struct cpufreq_driver integrator_driver = { - .verify = integrator_verify_policy, - .target = integrator_set_target, - .get = integrator_get, - .init = integrator_cpufreq_init, - .name = "integrator", -}; - -static int __init integrator_cpu_init(void) -{ - return cpufreq_register_driver(&integrator_driver); -} - -static void __exit integrator_cpu_exit(void) -{ - cpufreq_unregister_driver(&integrator_driver); -} - -MODULE_AUTHOR ("Russell M. King"); -MODULE_DESCRIPTION ("cpufreq driver for ARM Integrator CPUs"); -MODULE_LICENSE ("GPL"); - -module_init(integrator_cpu_init); -module_exit(integrator_cpu_exit); diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 07f2840ad805..ec4abe374573 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -1,15 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * intel_pstate.c: Native P state management for Intel processors * * (C) Copyright 2012 Intel Corporation * Author: Dirk Brandewie <dirk.j.brandewie@intel.com> - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; version 2 - * of the License. */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + #include <linux/kernel.h> #include <linux/kernel_stat.h> #include <linux/module.h> @@ -17,620 +15,3023 @@ #include <linux/hrtimer.h> #include <linux/tick.h> #include <linux/slab.h> -#include <linux/sched.h> +#include <linux/sched/cpufreq.h> +#include <linux/sched/smt.h> #include <linux/list.h> #include <linux/cpu.h> #include <linux/cpufreq.h> #include <linux/sysfs.h> #include <linux/types.h> #include <linux/fs.h> -#include <linux/debugfs.h> +#include <linux/acpi.h> +#include <linux/vmalloc.h> +#include <linux/pm_qos.h> +#include <linux/bitfield.h> #include <trace/events/power.h> +#include <linux/units.h> +#include <asm/cpu.h> #include <asm/div64.h> #include <asm/msr.h> #include <asm/cpu_device_id.h> +#include <asm/cpufeature.h> +#include <asm/intel-family.h> +#include "../drivers/thermal/intel/thermal_interrupt.h" + +#define INTEL_PSTATE_SAMPLING_INTERVAL (10 * NSEC_PER_MSEC) -#define SAMPLE_COUNT 3 +#define INTEL_CPUFREQ_TRANSITION_LATENCY 20000 +#define INTEL_CPUFREQ_TRANSITION_DELAY_HWP 5000 +#define INTEL_CPUFREQ_TRANSITION_DELAY 500 + +#ifdef CONFIG_ACPI +#include <acpi/processor.h> +#include <acpi/cppc_acpi.h> +#endif #define FRAC_BITS 8 #define int_tofp(X) ((int64_t)(X) << FRAC_BITS) #define fp_toint(X) ((X) >> FRAC_BITS) +#define ONE_EIGHTH_FP ((int64_t)1 << (FRAC_BITS - 3)) + +#define EXT_BITS 6 +#define EXT_FRAC_BITS (EXT_BITS + FRAC_BITS) +#define fp_ext_toint(X) ((X) >> EXT_FRAC_BITS) +#define int_ext_tofp(X) ((int64_t)(X) << EXT_FRAC_BITS) + static inline int32_t mul_fp(int32_t x, int32_t y) { return ((int64_t)x * (int64_t)y) >> FRAC_BITS; } -static inline int32_t div_fp(int32_t x, int32_t y) +static inline int32_t div_fp(s64 x, s64 y) +{ + return div64_s64((int64_t)x << FRAC_BITS, y); +} + +static inline int ceiling_fp(int32_t x) +{ + int mask, ret; + + ret = fp_toint(x); + mask = (1 << FRAC_BITS) - 1; + if (x & mask) + ret += 1; + return ret; +} + +static inline u64 mul_ext_fp(u64 x, u64 y) +{ + return (x * y) >> EXT_FRAC_BITS; +} + +static inline u64 div_ext_fp(u64 x, u64 y) { - return div_s64((int64_t)x << FRAC_BITS, (int64_t)y); + return div64_u64(x << EXT_FRAC_BITS, y); } +/** + * struct sample - Store performance sample + * @core_avg_perf: Ratio of APERF/MPERF which is the actual average + * performance during last sample period + * @busy_scaled: Scaled busy value which is used to calculate next + * P state. This can be different than core_avg_perf + * to account for cpu idle period + * @aperf: Difference of actual performance frequency clock count + * read from APERF MSR between last and current sample + * @mperf: Difference of maximum performance frequency clock count + * read from MPERF MSR between last and current sample + * @tsc: Difference of time stamp counter between last and + * current sample + * @time: Current time from scheduler + * + * This structure is used in the cpudata structure to store performance sample + * data for choosing next P State. + */ struct sample { - int core_pct_busy; + int32_t core_avg_perf; + int32_t busy_scaled; u64 aperf; u64 mperf; - int freq; + u64 tsc; + u64 time; }; +/** + * struct pstate_data - Store P state data + * @current_pstate: Current requested P state + * @min_pstate: Min P state possible for this platform + * @max_pstate: Max P state possible for this platform + * @max_pstate_physical:This is physical Max P state for a processor + * This can be higher than the max_pstate which can + * be limited by platform thermal design power limits + * @perf_ctl_scaling: PERF_CTL P-state to frequency scaling factor + * @scaling: Scaling factor between performance and frequency + * @turbo_pstate: Max Turbo P state possible for this platform + * @min_freq: @min_pstate frequency in cpufreq units + * @max_freq: @max_pstate frequency in cpufreq units + * @turbo_freq: @turbo_pstate frequency in cpufreq units + * + * Stores the per cpu model P state limits and current P state. + */ struct pstate_data { int current_pstate; int min_pstate; int max_pstate; + int max_pstate_physical; + int perf_ctl_scaling; + int scaling; int turbo_pstate; + unsigned int min_freq; + unsigned int max_freq; + unsigned int turbo_freq; +}; + +/** + * struct vid_data - Stores voltage information data + * @min: VID data for this platform corresponding to + * the lowest P state + * @max: VID data corresponding to the highest P State. + * @turbo: VID data for turbo P state + * @ratio: Ratio of (vid max - vid min) / + * (max P state - Min P State) + * + * Stores the voltage data for DVFS (Dynamic Voltage and Frequency Scaling) + * This data is used in Atom platforms, where in addition to target P state, + * the voltage data needs to be specified to select next P State. + */ +struct vid_data { + int min; + int max; + int turbo; + int32_t ratio; }; -struct _pid { - int setpoint; - int32_t integral; - int32_t p_gain; - int32_t i_gain; - int32_t d_gain; - int deadband; - int last_err; +/** + * struct global_params - Global parameters, mostly tunable via sysfs. + * @no_turbo: Whether or not to use turbo P-states. + * @turbo_disabled: Whether or not turbo P-states are available at all, + * based on the MSR_IA32_MISC_ENABLE value and whether or + * not the maximum reported turbo P-state is different from + * the maximum reported non-turbo one. + * @min_perf_pct: Minimum capacity limit in percent of the maximum turbo + * P-state capacity. + * @max_perf_pct: Maximum capacity limit in percent of the maximum turbo + * P-state capacity. + */ +struct global_params { + bool no_turbo; + bool turbo_disabled; + int max_perf_pct; + int min_perf_pct; }; +/** + * struct cpudata - Per CPU instance data storage + * @cpu: CPU number for this instance data + * @policy: CPUFreq policy value + * @update_util: CPUFreq utility callback information + * @update_util_set: CPUFreq utility callback is set + * @iowait_boost: iowait-related boost fraction + * @last_update: Time of the last update. + * @pstate: Stores P state limits for this CPU + * @vid: Stores VID limits for this CPU + * @last_sample_time: Last Sample time + * @aperf_mperf_shift: APERF vs MPERF counting frequency difference + * @prev_aperf: Last APERF value read from APERF MSR + * @prev_mperf: Last MPERF value read from MPERF MSR + * @prev_tsc: Last timestamp counter (TSC) value + * @sample: Storage for storing last Sample data + * @min_perf_ratio: Minimum capacity in terms of PERF or HWP ratios + * @max_perf_ratio: Maximum capacity in terms of PERF or HWP ratios + * @acpi_perf_data: Stores ACPI perf information read from _PSS + * @valid_pss_table: Set to true for valid ACPI _PSS entries found + * @epp_powersave: Last saved HWP energy performance preference + * (EPP) or energy performance bias (EPB), + * when policy switched to performance + * @epp_policy: Last saved policy used to set EPP/EPB + * @epp_default: Power on default HWP energy performance + * preference/bias + * @epp_cached: Cached HWP energy-performance preference value + * @hwp_req_cached: Cached value of the last HWP Request MSR + * @hwp_cap_cached: Cached value of the last HWP Capabilities MSR + * @last_io_update: Last time when IO wake flag was set + * @capacity_perf: Highest perf used for scale invariance + * @sched_flags: Store scheduler flags for possible cross CPU update + * @hwp_boost_min: Last HWP boosted min performance + * @suspended: Whether or not the driver has been suspended. + * @pd_registered: Set when a perf domain is registered for this CPU. + * @hwp_notify_work: workqueue for HWP notifications. + * + * This structure stores per CPU instance data for all CPUs. + */ struct cpudata { int cpu; - char name[64]; - - struct timer_list timer; + unsigned int policy; + struct update_util_data update_util; + bool update_util_set; - struct pstate_adjust_policy *pstate_policy; struct pstate_data pstate; - struct _pid pid; - - int min_pstate_count; + struct vid_data vid; + u64 last_update; + u64 last_sample_time; + u64 aperf_mperf_shift; u64 prev_aperf; u64 prev_mperf; - int sample_ptr; - struct sample samples[SAMPLE_COUNT]; + u64 prev_tsc; + struct sample sample; + int32_t min_perf_ratio; + int32_t max_perf_ratio; +#ifdef CONFIG_ACPI + struct acpi_processor_performance acpi_perf_data; + bool valid_pss_table; +#endif + unsigned int iowait_boost; + s16 epp_powersave; + s16 epp_policy; + s16 epp_default; + s16 epp_cached; + u64 hwp_req_cached; + u64 hwp_cap_cached; + u64 last_io_update; + unsigned int capacity_perf; + unsigned int sched_flags; + u32 hwp_boost_min; + bool suspended; +#ifdef CONFIG_ENERGY_MODEL + bool pd_registered; +#endif + struct delayed_work hwp_notify_work; }; static struct cpudata **all_cpu_data; -struct pstate_adjust_policy { - int sample_rate_ms; - int deadband; - int setpoint; - int p_gain_pct; - int d_gain_pct; - int i_gain_pct; -}; -static struct pstate_adjust_policy default_policy = { - .sample_rate_ms = 10, - .deadband = 0, - .setpoint = 109, - .p_gain_pct = 17, - .d_gain_pct = 0, - .i_gain_pct = 4, +/** + * struct pstate_funcs - Per CPU model specific callbacks + * @get_max: Callback to get maximum non turbo effective P state + * @get_max_physical: Callback to get maximum non turbo physical P state + * @get_min: Callback to get minimum P state + * @get_turbo: Callback to get turbo P state + * @get_scaling: Callback to get frequency scaling factor + * @get_cpu_scaling: Get frequency scaling factor for a given cpu + * @get_aperf_mperf_shift: Callback to get the APERF vs MPERF frequency difference + * @get_val: Callback to convert P state to actual MSR write value + * @get_vid: Callback to get VID data for Atom platforms + * + * Core and Atom CPU models have different way to get P State limits. This + * structure is used to store those callbacks. + */ +struct pstate_funcs { + int (*get_max)(int cpu); + int (*get_max_physical)(int cpu); + int (*get_min)(int cpu); + int (*get_turbo)(int cpu); + int (*get_scaling)(void); + int (*get_cpu_scaling)(int cpu); + int (*get_aperf_mperf_shift)(void); + u64 (*get_val)(struct cpudata*, int pstate); + void (*get_vid)(struct cpudata *); }; -struct perf_limits { - int no_turbo; - int max_perf_pct; - int min_perf_pct; - int32_t max_perf; - int32_t min_perf; - int max_policy_pct; - int max_sysfs_pct; +static struct pstate_funcs pstate_funcs __read_mostly; + +static bool hwp_active __ro_after_init; +static int hwp_mode_bdw __ro_after_init; +static bool per_cpu_limits __ro_after_init; +static bool hwp_forced __ro_after_init; +static bool hwp_boost __read_mostly; +static bool hwp_is_hybrid; + +static struct cpufreq_driver *intel_pstate_driver __read_mostly; + +#define INTEL_PSTATE_CORE_SCALING 100000 +#define HYBRID_SCALING_FACTOR_ADL 78741 +#define HYBRID_SCALING_FACTOR_MTL 80000 +#define HYBRID_SCALING_FACTOR_LNL 86957 + +static int hybrid_scaling_factor; + +static inline int core_get_scaling(void) +{ + return INTEL_PSTATE_CORE_SCALING; +} + +#ifdef CONFIG_ACPI +static bool acpi_ppc; +#endif + +static struct global_params global; + +static DEFINE_MUTEX(intel_pstate_driver_lock); +static DEFINE_MUTEX(intel_pstate_limits_lock); + +#ifdef CONFIG_ACPI + +static bool intel_pstate_acpi_pm_profile_server(void) +{ + if (acpi_gbl_FADT.preferred_profile == PM_ENTERPRISE_SERVER || + acpi_gbl_FADT.preferred_profile == PM_PERFORMANCE_SERVER) + return true; + + return false; +} + +static bool intel_pstate_get_ppc_enable_status(void) +{ + if (intel_pstate_acpi_pm_profile_server()) + return true; + + return acpi_ppc; +} + +#ifdef CONFIG_ACPI_CPPC_LIB + +/* The work item is needed to avoid CPU hotplug locking issues */ +static void intel_pstste_sched_itmt_work_fn(struct work_struct *work) +{ + sched_set_itmt_support(); +} + +static DECLARE_WORK(sched_itmt_work, intel_pstste_sched_itmt_work_fn); + +#define CPPC_MAX_PERF U8_MAX + +static void intel_pstate_set_itmt_prio(int cpu) +{ + struct cppc_perf_caps cppc_perf; + static u32 max_highest_perf = 0, min_highest_perf = U32_MAX; + int ret; + + ret = cppc_get_perf_caps(cpu, &cppc_perf); + /* + * If CPPC is not available, fall back to MSR_HWP_CAPABILITIES bits [8:0]. + * + * Also, on some systems with overclocking enabled, CPPC.highest_perf is + * hardcoded to 0xff, so CPPC.highest_perf cannot be used to enable ITMT. + * Fall back to MSR_HWP_CAPABILITIES then too. + */ + if (ret || cppc_perf.highest_perf == CPPC_MAX_PERF) + cppc_perf.highest_perf = HWP_HIGHEST_PERF(READ_ONCE(all_cpu_data[cpu]->hwp_cap_cached)); + + /* + * The priorities can be set regardless of whether or not + * sched_set_itmt_support(true) has been called and it is valid to + * update them at any time after it has been called. + */ + sched_set_itmt_core_prio(cppc_perf.highest_perf, cpu); + + if (max_highest_perf <= min_highest_perf) { + if (cppc_perf.highest_perf > max_highest_perf) + max_highest_perf = cppc_perf.highest_perf; + + if (cppc_perf.highest_perf < min_highest_perf) + min_highest_perf = cppc_perf.highest_perf; + + if (max_highest_perf > min_highest_perf) { + /* + * This code can be run during CPU online under the + * CPU hotplug locks, so sched_set_itmt_support() + * cannot be called from here. Queue up a work item + * to invoke it. + */ + schedule_work(&sched_itmt_work); + } + } +} + +static int intel_pstate_get_cppc_guaranteed(int cpu) +{ + struct cppc_perf_caps cppc_perf; + int ret; + + ret = cppc_get_perf_caps(cpu, &cppc_perf); + if (ret) + return ret; + + if (cppc_perf.guaranteed_perf) + return cppc_perf.guaranteed_perf; + + return cppc_perf.nominal_perf; +} + +static int intel_pstate_cppc_get_scaling(int cpu) +{ + struct cppc_perf_caps cppc_perf; + + /* + * Compute the perf-to-frequency scaling factor for the given CPU if + * possible, unless it would be 0. + */ + if (!cppc_get_perf_caps(cpu, &cppc_perf) && + cppc_perf.nominal_perf && cppc_perf.nominal_freq) + return div_u64(cppc_perf.nominal_freq * KHZ_PER_MHZ, + cppc_perf.nominal_perf); + + return core_get_scaling(); +} + +#else /* CONFIG_ACPI_CPPC_LIB */ +static inline void intel_pstate_set_itmt_prio(int cpu) +{ +} +#endif /* CONFIG_ACPI_CPPC_LIB */ + +static void intel_pstate_init_acpi_perf_limits(struct cpufreq_policy *policy) +{ + struct cpudata *cpu; + int ret; + int i; + + if (hwp_active) { + intel_pstate_set_itmt_prio(policy->cpu); + return; + } + + if (!intel_pstate_get_ppc_enable_status()) + return; + + cpu = all_cpu_data[policy->cpu]; + + ret = acpi_processor_register_performance(&cpu->acpi_perf_data, + policy->cpu); + if (ret) + return; + + /* + * Check if the control value in _PSS is for PERF_CTL MSR, which should + * guarantee that the states returned by it map to the states in our + * list directly. + */ + if (cpu->acpi_perf_data.control_register.space_id != + ACPI_ADR_SPACE_FIXED_HARDWARE) + goto err; + + /* + * If there is only one entry _PSS, simply ignore _PSS and continue as + * usual without taking _PSS into account + */ + if (cpu->acpi_perf_data.state_count < 2) + goto err; + + pr_debug("CPU%u - ACPI _PSS perf data\n", policy->cpu); + for (i = 0; i < cpu->acpi_perf_data.state_count; i++) { + pr_debug(" %cP%d: %u MHz, %u mW, 0x%x\n", + (i == cpu->acpi_perf_data.state ? '*' : ' '), i, + (u32) cpu->acpi_perf_data.states[i].core_frequency, + (u32) cpu->acpi_perf_data.states[i].power, + (u32) cpu->acpi_perf_data.states[i].control); + } + + cpu->valid_pss_table = true; + pr_debug("_PPC limits will be enforced\n"); + + return; + + err: + cpu->valid_pss_table = false; + acpi_processor_unregister_performance(policy->cpu); +} + +static void intel_pstate_exit_perf_limits(struct cpufreq_policy *policy) +{ + struct cpudata *cpu; + + cpu = all_cpu_data[policy->cpu]; + if (!cpu->valid_pss_table) + return; + + acpi_processor_unregister_performance(policy->cpu); +} +#else /* CONFIG_ACPI */ +static inline void intel_pstate_init_acpi_perf_limits(struct cpufreq_policy *policy) +{ +} + +static inline void intel_pstate_exit_perf_limits(struct cpufreq_policy *policy) +{ +} + +static inline bool intel_pstate_acpi_pm_profile_server(void) +{ + return false; +} +#endif /* CONFIG_ACPI */ + +#ifndef CONFIG_ACPI_CPPC_LIB +static inline int intel_pstate_get_cppc_guaranteed(int cpu) +{ + return -ENOTSUPP; +} + +static int intel_pstate_cppc_get_scaling(int cpu) +{ + return core_get_scaling(); +} +#endif /* CONFIG_ACPI_CPPC_LIB */ + +static int intel_pstate_freq_to_hwp_rel(struct cpudata *cpu, int freq, + unsigned int relation) +{ + if (freq == cpu->pstate.turbo_freq) + return cpu->pstate.turbo_pstate; + + if (freq == cpu->pstate.max_freq) + return cpu->pstate.max_pstate; + + switch (relation) { + case CPUFREQ_RELATION_H: + return freq / cpu->pstate.scaling; + case CPUFREQ_RELATION_C: + return DIV_ROUND_CLOSEST(freq, cpu->pstate.scaling); + } + + return DIV_ROUND_UP(freq, cpu->pstate.scaling); +} + +static int intel_pstate_freq_to_hwp(struct cpudata *cpu, int freq) +{ + return intel_pstate_freq_to_hwp_rel(cpu, freq, CPUFREQ_RELATION_L); +} + +/** + * intel_pstate_hybrid_hwp_adjust - Calibrate HWP performance levels. + * @cpu: Target CPU. + * + * On hybrid processors, HWP may expose more performance levels than there are + * P-states accessible through the PERF_CTL interface. If that happens, the + * scaling factor between HWP performance levels and CPU frequency will be less + * than the scaling factor between P-state values and CPU frequency. + * + * In that case, adjust the CPU parameters used in computations accordingly. + */ +static void intel_pstate_hybrid_hwp_adjust(struct cpudata *cpu) +{ + int perf_ctl_max_phys = cpu->pstate.max_pstate_physical; + int perf_ctl_scaling = cpu->pstate.perf_ctl_scaling; + int perf_ctl_turbo = pstate_funcs.get_turbo(cpu->cpu); + int scaling = cpu->pstate.scaling; + int freq; + + pr_debug("CPU%d: PERF_CTL max_phys = %d\n", cpu->cpu, perf_ctl_max_phys); + pr_debug("CPU%d: PERF_CTL turbo = %d\n", cpu->cpu, perf_ctl_turbo); + pr_debug("CPU%d: PERF_CTL scaling = %d\n", cpu->cpu, perf_ctl_scaling); + pr_debug("CPU%d: HWP_CAP guaranteed = %d\n", cpu->cpu, cpu->pstate.max_pstate); + pr_debug("CPU%d: HWP_CAP highest = %d\n", cpu->cpu, cpu->pstate.turbo_pstate); + pr_debug("CPU%d: HWP-to-frequency scaling factor: %d\n", cpu->cpu, scaling); + + if (scaling == perf_ctl_scaling) + return; + + hwp_is_hybrid = true; + + cpu->pstate.turbo_freq = rounddown(cpu->pstate.turbo_pstate * scaling, + perf_ctl_scaling); + cpu->pstate.max_freq = rounddown(cpu->pstate.max_pstate * scaling, + perf_ctl_scaling); + + freq = perf_ctl_max_phys * perf_ctl_scaling; + cpu->pstate.max_pstate_physical = intel_pstate_freq_to_hwp(cpu, freq); + + freq = cpu->pstate.min_pstate * perf_ctl_scaling; + cpu->pstate.min_freq = freq; + /* + * Cast the min P-state value retrieved via pstate_funcs.get_min() to + * the effective range of HWP performance levels. + */ + cpu->pstate.min_pstate = intel_pstate_freq_to_hwp(cpu, freq); +} + +static bool turbo_is_disabled(void) +{ + u64 misc_en; + + rdmsrq(MSR_IA32_MISC_ENABLE, misc_en); + + return !!(misc_en & MSR_IA32_MISC_ENABLE_TURBO_DISABLE); +} + +static int min_perf_pct_min(void) +{ + struct cpudata *cpu = all_cpu_data[0]; + int turbo_pstate = cpu->pstate.turbo_pstate; + + return turbo_pstate ? + (cpu->pstate.min_pstate * 100 / turbo_pstate) : 0; +} + +static s16 intel_pstate_get_epp(struct cpudata *cpu_data, u64 hwp_req_data) +{ + s16 epp = -EOPNOTSUPP; + + if (boot_cpu_has(X86_FEATURE_HWP_EPP)) { + /* + * When hwp_req_data is 0, means that caller didn't read + * MSR_HWP_REQUEST, so need to read and get EPP. + */ + if (!hwp_req_data) { + epp = rdmsrq_on_cpu(cpu_data->cpu, MSR_HWP_REQUEST, + &hwp_req_data); + if (epp) + return epp; + } + epp = (hwp_req_data >> 24) & 0xff; + } + + return epp; +} + +/* + * EPP display strings corresponding to EPP index in the + * energy_perf_strings[] + * index String + *------------------------------------- + * 0 default + * 1 performance + * 2 balance_performance + * 3 balance_power + * 4 power + */ + +enum energy_perf_value_index { + EPP_INDEX_DEFAULT = 0, + EPP_INDEX_PERFORMANCE, + EPP_INDEX_BALANCE_PERFORMANCE, + EPP_INDEX_BALANCE_POWERSAVE, + EPP_INDEX_POWERSAVE, }; -static struct perf_limits limits = { - .no_turbo = 0, - .max_perf_pct = 100, - .max_perf = int_tofp(1), - .min_perf_pct = 0, - .min_perf = 0, - .max_policy_pct = 100, - .max_sysfs_pct = 100, +static const char * const energy_perf_strings[] = { + [EPP_INDEX_DEFAULT] = "default", + [EPP_INDEX_PERFORMANCE] = "performance", + [EPP_INDEX_BALANCE_PERFORMANCE] = "balance_performance", + [EPP_INDEX_BALANCE_POWERSAVE] = "balance_power", + [EPP_INDEX_POWERSAVE] = "power", + NULL +}; +static unsigned int epp_values[] = { + [EPP_INDEX_DEFAULT] = 0, /* Unused index */ + [EPP_INDEX_PERFORMANCE] = HWP_EPP_PERFORMANCE, + [EPP_INDEX_BALANCE_PERFORMANCE] = HWP_EPP_BALANCE_PERFORMANCE, + [EPP_INDEX_BALANCE_POWERSAVE] = HWP_EPP_BALANCE_POWERSAVE, + [EPP_INDEX_POWERSAVE] = HWP_EPP_POWERSAVE, }; -static inline void pid_reset(struct _pid *pid, int setpoint, int busy, - int deadband, int integral) { - pid->setpoint = setpoint; - pid->deadband = deadband; - pid->integral = int_tofp(integral); - pid->last_err = setpoint - busy; +static int intel_pstate_get_energy_pref_index(struct cpudata *cpu_data, int *raw_epp) +{ + s16 epp; + int index = -EINVAL; + + *raw_epp = 0; + epp = intel_pstate_get_epp(cpu_data, 0); + if (epp < 0) + return epp; + + if (boot_cpu_has(X86_FEATURE_HWP_EPP)) { + if (epp == epp_values[EPP_INDEX_PERFORMANCE]) + return EPP_INDEX_PERFORMANCE; + if (epp == epp_values[EPP_INDEX_BALANCE_PERFORMANCE]) + return EPP_INDEX_BALANCE_PERFORMANCE; + if (epp == epp_values[EPP_INDEX_BALANCE_POWERSAVE]) + return EPP_INDEX_BALANCE_POWERSAVE; + if (epp == epp_values[EPP_INDEX_POWERSAVE]) + return EPP_INDEX_POWERSAVE; + *raw_epp = epp; + return 0; + } else if (boot_cpu_has(X86_FEATURE_EPB)) { + /* + * Range: + * 0x00-0x03 : Performance + * 0x04-0x07 : Balance performance + * 0x08-0x0B : Balance power + * 0x0C-0x0F : Power + * The EPB is a 4 bit value, but our ranges restrict the + * value which can be set. Here only using top two bits + * effectively. + */ + index = (epp >> 2) + 1; + } + + return index; } -static inline void pid_p_gain_set(struct _pid *pid, int percent) +static int intel_pstate_set_epp(struct cpudata *cpu, u32 epp) { - pid->p_gain = div_fp(int_tofp(percent), int_tofp(100)); + int ret; + + /* + * Use the cached HWP Request MSR value, because in the active mode the + * register itself may be updated by intel_pstate_hwp_boost_up() or + * intel_pstate_hwp_boost_down() at any time. + */ + u64 value = READ_ONCE(cpu->hwp_req_cached); + + value &= ~GENMASK_ULL(31, 24); + value |= (u64)epp << 24; + /* + * The only other updater of hwp_req_cached in the active mode, + * intel_pstate_hwp_set(), is called under the same lock as this + * function, so it cannot run in parallel with the update below. + */ + WRITE_ONCE(cpu->hwp_req_cached, value); + ret = wrmsrq_on_cpu(cpu->cpu, MSR_HWP_REQUEST, value); + if (!ret) + cpu->epp_cached = epp; + + return ret; } -static inline void pid_i_gain_set(struct _pid *pid, int percent) +static int intel_pstate_set_energy_pref_index(struct cpudata *cpu_data, + int pref_index, bool use_raw, + u32 raw_epp) { - pid->i_gain = div_fp(int_tofp(percent), int_tofp(100)); + int epp = -EINVAL; + int ret = -EOPNOTSUPP; + + if (!pref_index) + epp = cpu_data->epp_default; + + if (boot_cpu_has(X86_FEATURE_HWP_EPP)) { + if (use_raw) + epp = raw_epp; + else if (epp == -EINVAL) + epp = epp_values[pref_index]; + + /* + * To avoid confusion, refuse to set EPP to any values different + * from 0 (performance) if the current policy is "performance", + * because those values would be overridden. + */ + if (epp > 0 && cpu_data->policy == CPUFREQ_POLICY_PERFORMANCE) + return -EBUSY; + + ret = intel_pstate_set_epp(cpu_data, epp); + } + + return ret; } -static inline void pid_d_gain_set(struct _pid *pid, int percent) +static ssize_t show_energy_performance_available_preferences( + struct cpufreq_policy *policy, char *buf) { + int i = 0; + int ret = 0; + + while (energy_perf_strings[i] != NULL) + ret += sprintf(&buf[ret], "%s ", energy_perf_strings[i++]); - pid->d_gain = div_fp(int_tofp(percent), int_tofp(100)); + ret += sprintf(&buf[ret], "\n"); + + return ret; } -static signed int pid_calc(struct _pid *pid, int busy) +cpufreq_freq_attr_ro(energy_performance_available_preferences); + +static struct cpufreq_driver intel_pstate; + +static ssize_t store_energy_performance_preference( + struct cpufreq_policy *policy, const char *buf, size_t count) { - signed int err, result; - int32_t pterm, dterm, fp_error; - int32_t integral_limit; + struct cpudata *cpu = all_cpu_data[policy->cpu]; + char str_preference[21]; + bool raw = false; + ssize_t ret; + u32 epp = 0; - err = pid->setpoint - busy; - fp_error = int_tofp(err); + ret = sscanf(buf, "%20s", str_preference); + if (ret != 1) + return -EINVAL; - if (abs(err) <= pid->deadband) - return 0; + ret = match_string(energy_perf_strings, -1, str_preference); + if (ret < 0) { + if (!boot_cpu_has(X86_FEATURE_HWP_EPP)) + return ret; - pterm = mul_fp(pid->p_gain, fp_error); + ret = kstrtouint(buf, 10, &epp); + if (ret) + return ret; - pid->integral += fp_error; + if (epp > 255) + return -EINVAL; - /* limit the integral term */ - integral_limit = int_tofp(30); - if (pid->integral > integral_limit) - pid->integral = integral_limit; - if (pid->integral < -integral_limit) - pid->integral = -integral_limit; + raw = true; + } - dterm = mul_fp(pid->d_gain, (err - pid->last_err)); - pid->last_err = err; + /* + * This function runs with the policy R/W semaphore held, which + * guarantees that the driver pointer will not change while it is + * running. + */ + if (!intel_pstate_driver) + return -EAGAIN; + + mutex_lock(&intel_pstate_limits_lock); + + if (intel_pstate_driver == &intel_pstate) { + ret = intel_pstate_set_energy_pref_index(cpu, ret, raw, epp); + } else { + /* + * In the passive mode the governor needs to be stopped on the + * target CPU before the EPP update and restarted after it, + * which is super-heavy-weight, so make sure it is worth doing + * upfront. + */ + if (!raw) + epp = ret ? epp_values[ret] : cpu->epp_default; + + if (cpu->epp_cached != epp) { + int err; + + cpufreq_stop_governor(policy); + ret = intel_pstate_set_epp(cpu, epp); + err = cpufreq_start_governor(policy); + if (!ret) + ret = err; + } else { + ret = 0; + } + } - result = pterm + mul_fp(pid->integral, pid->i_gain) + dterm; + mutex_unlock(&intel_pstate_limits_lock); - return (signed int)fp_toint(result); + return ret ?: count; } -static inline void intel_pstate_busy_pid_reset(struct cpudata *cpu) +static ssize_t show_energy_performance_preference( + struct cpufreq_policy *policy, char *buf) { - pid_p_gain_set(&cpu->pid, cpu->pstate_policy->p_gain_pct); - pid_d_gain_set(&cpu->pid, cpu->pstate_policy->d_gain_pct); - pid_i_gain_set(&cpu->pid, cpu->pstate_policy->i_gain_pct); + struct cpudata *cpu_data = all_cpu_data[policy->cpu]; + int preference, raw_epp; + + preference = intel_pstate_get_energy_pref_index(cpu_data, &raw_epp); + if (preference < 0) + return preference; - pid_reset(&cpu->pid, - cpu->pstate_policy->setpoint, - 100, - cpu->pstate_policy->deadband, - 0); + if (raw_epp) + return sprintf(buf, "%d\n", raw_epp); + else + return sprintf(buf, "%s\n", energy_perf_strings[preference]); } -static inline void intel_pstate_reset_all_pid(void) +cpufreq_freq_attr_rw(energy_performance_preference); + +static ssize_t show_base_frequency(struct cpufreq_policy *policy, char *buf) { - unsigned int cpu; - for_each_online_cpu(cpu) { - if (all_cpu_data[cpu]) - intel_pstate_busy_pid_reset(all_cpu_data[cpu]); + struct cpudata *cpu = all_cpu_data[policy->cpu]; + int ratio, freq; + + ratio = intel_pstate_get_cppc_guaranteed(policy->cpu); + if (ratio <= 0) { + u64 cap; + + rdmsrq_on_cpu(policy->cpu, MSR_HWP_CAPABILITIES, &cap); + ratio = HWP_GUARANTEED_PERF(cap); } + + freq = ratio * cpu->pstate.scaling; + if (cpu->pstate.scaling != cpu->pstate.perf_ctl_scaling) + freq = rounddown(freq, cpu->pstate.perf_ctl_scaling); + + return sprintf(buf, "%d\n", freq); +} + +cpufreq_freq_attr_ro(base_frequency); + +enum hwp_cpufreq_attr_index { + HWP_BASE_FREQUENCY_INDEX = 0, + HWP_PERFORMANCE_PREFERENCE_INDEX, + HWP_PERFORMANCE_AVAILABLE_PREFERENCES_INDEX, + HWP_CPUFREQ_ATTR_COUNT, +}; + +static struct freq_attr *hwp_cpufreq_attrs[] = { + [HWP_BASE_FREQUENCY_INDEX] = &base_frequency, + [HWP_PERFORMANCE_PREFERENCE_INDEX] = &energy_performance_preference, + [HWP_PERFORMANCE_AVAILABLE_PREFERENCES_INDEX] = + &energy_performance_available_preferences, + [HWP_CPUFREQ_ATTR_COUNT] = NULL, +}; + +static u8 hybrid_get_cpu_type(unsigned int cpu) +{ + return cpu_data(cpu).topo.intel_type; } -/************************** debugfs begin ************************/ -static int pid_param_set(void *data, u64 val) +static bool no_cas __ro_after_init; + +static struct cpudata *hybrid_max_perf_cpu __read_mostly; +/* + * Protects hybrid_max_perf_cpu, the capacity_perf fields in struct cpudata, + * and the x86 arch scale-invariance information from concurrent updates. + */ +static DEFINE_MUTEX(hybrid_capacity_lock); + +#ifdef CONFIG_ENERGY_MODEL +#define HYBRID_EM_STATE_COUNT 4 + +static int hybrid_active_power(struct device *dev, unsigned long *power, + unsigned long *freq) { - *(u32 *)data = val; - intel_pstate_reset_all_pid(); + /* + * Create four "states" corresponding to 40%, 60%, 80%, and 100% of the + * full capacity. + * + * For this purpose, return the "frequency" of 2 for the first + * performance level and otherwise leave the value set by the caller. + */ + if (!*freq) + *freq = 2; + + /* No power information. */ + *power = EM_MAX_POWER; + return 0; } -static int pid_param_get(void *data, u64 *val) + +static bool hybrid_has_l3(unsigned int cpu) +{ + struct cpu_cacheinfo *cacheinfo = get_cpu_cacheinfo(cpu); + unsigned int i; + + if (!cacheinfo) + return false; + + for (i = 0; i < cacheinfo->num_leaves; i++) { + if (cacheinfo->info_list[i].level == 3) + return true; + } + + return false; +} + +static int hybrid_get_cost(struct device *dev, unsigned long freq, + unsigned long *cost) { - *val = *(u32 *)data; + /* Facilitate load balancing between CPUs of the same type. */ + *cost = freq; + /* + * Adjust the cost depending on CPU type. + * + * The idea is to start loading up LPE-cores before E-cores and start + * to populate E-cores when LPE-cores are utilized above 60% of the + * capacity. Similarly, P-cores start to be populated when E-cores are + * utilized above 60% of the capacity. + */ + if (hybrid_get_cpu_type(dev->id) == INTEL_CPU_TYPE_ATOM) { + if (hybrid_has_l3(dev->id)) /* E-core */ + *cost += 1; + } else { /* P-core */ + *cost += 2; + } + return 0; } -DEFINE_SIMPLE_ATTRIBUTE(fops_pid_param, pid_param_get, - pid_param_set, "%llu\n"); -struct pid_param { - char *name; - void *value; -}; +static bool hybrid_register_perf_domain(unsigned int cpu) +{ + static const struct em_data_callback cb + = EM_ADV_DATA_CB(hybrid_active_power, hybrid_get_cost); + struct cpudata *cpudata = all_cpu_data[cpu]; + struct device *cpu_dev; -static struct pid_param pid_files[] = { - {"sample_rate_ms", &default_policy.sample_rate_ms}, - {"d_gain_pct", &default_policy.d_gain_pct}, - {"i_gain_pct", &default_policy.i_gain_pct}, - {"deadband", &default_policy.deadband}, - {"setpoint", &default_policy.setpoint}, - {"p_gain_pct", &default_policy.p_gain_pct}, - {NULL, NULL} -}; + /* + * Registering EM perf domains without enabling asymmetric CPU capacity + * support is not really useful and one domain should not be registered + * more than once. + */ + if (!hybrid_max_perf_cpu || cpudata->pd_registered) + return false; + + cpu_dev = get_cpu_device(cpu); + if (!cpu_dev) + return false; + + if (em_dev_register_pd_no_update(cpu_dev, HYBRID_EM_STATE_COUNT, &cb, + cpumask_of(cpu), false)) + return false; -static struct dentry *debugfs_parent; -static void intel_pstate_debug_expose_params(void) + cpudata->pd_registered = true; + + return true; +} + +static void hybrid_register_all_perf_domains(void) { - int i = 0; + unsigned int cpu; + + for_each_online_cpu(cpu) + hybrid_register_perf_domain(cpu); +} + +static void hybrid_update_perf_domain(struct cpudata *cpu) +{ + if (cpu->pd_registered) + em_adjust_cpu_capacity(cpu->cpu); +} +#else /* !CONFIG_ENERGY_MODEL */ +static inline bool hybrid_register_perf_domain(unsigned int cpu) { return false; } +static inline void hybrid_register_all_perf_domains(void) {} +static inline void hybrid_update_perf_domain(struct cpudata *cpu) {} +#endif /* CONFIG_ENERGY_MODEL */ + +static void hybrid_set_cpu_capacity(struct cpudata *cpu) +{ + arch_set_cpu_capacity(cpu->cpu, cpu->capacity_perf, + hybrid_max_perf_cpu->capacity_perf, + cpu->capacity_perf, + cpu->pstate.max_pstate_physical); + hybrid_update_perf_domain(cpu); + + topology_set_cpu_scale(cpu->cpu, arch_scale_cpu_capacity(cpu->cpu)); - debugfs_parent = debugfs_create_dir("pstate_snb", NULL); - if (IS_ERR_OR_NULL(debugfs_parent)) + pr_debug("CPU%d: capacity perf = %u, base perf = %u, sys max perf = %u\n", + cpu->cpu, cpu->capacity_perf, cpu->pstate.max_pstate_physical, + hybrid_max_perf_cpu->capacity_perf); +} + +static void hybrid_clear_cpu_capacity(unsigned int cpunum) +{ + arch_set_cpu_capacity(cpunum, 1, 1, 1, 1); +} + +static void hybrid_get_capacity_perf(struct cpudata *cpu) +{ + if (READ_ONCE(global.no_turbo)) { + cpu->capacity_perf = cpu->pstate.max_pstate_physical; return; - while (pid_files[i].name) { - debugfs_create_file(pid_files[i].name, 0660, - debugfs_parent, pid_files[i].value, - &fops_pid_param); - i++; + } + + cpu->capacity_perf = HWP_HIGHEST_PERF(READ_ONCE(cpu->hwp_cap_cached)); +} + +static void hybrid_set_capacity_of_cpus(void) +{ + int cpunum; + + for_each_online_cpu(cpunum) { + struct cpudata *cpu = all_cpu_data[cpunum]; + + if (cpu) + hybrid_set_cpu_capacity(cpu); + } +} + +static void hybrid_update_cpu_capacity_scaling(void) +{ + struct cpudata *max_perf_cpu = NULL; + unsigned int max_cap_perf = 0; + int cpunum; + + for_each_online_cpu(cpunum) { + struct cpudata *cpu = all_cpu_data[cpunum]; + + if (!cpu) + continue; + + /* + * During initialization, CPU performance at full capacity needs + * to be determined. + */ + if (!hybrid_max_perf_cpu) + hybrid_get_capacity_perf(cpu); + + /* + * If hybrid_max_perf_cpu is not NULL at this point, it is + * being replaced, so don't take it into account when looking + * for the new one. + */ + if (cpu == hybrid_max_perf_cpu) + continue; + + if (cpu->capacity_perf > max_cap_perf) { + max_cap_perf = cpu->capacity_perf; + max_perf_cpu = cpu; + } + } + + if (max_perf_cpu) { + hybrid_max_perf_cpu = max_perf_cpu; + hybrid_set_capacity_of_cpus(); + } else { + pr_info("Found no CPUs with nonzero maximum performance\n"); + /* Revert to the flat CPU capacity structure. */ + for_each_online_cpu(cpunum) + hybrid_clear_cpu_capacity(cpunum); } } -/************************** debugfs end ************************/ +static void __hybrid_refresh_cpu_capacity_scaling(void) +{ + hybrid_max_perf_cpu = NULL; + hybrid_update_cpu_capacity_scaling(); +} + +static void hybrid_refresh_cpu_capacity_scaling(void) +{ + guard(mutex)(&hybrid_capacity_lock); + + __hybrid_refresh_cpu_capacity_scaling(); + /* + * Perf domains are not registered before setting hybrid_max_perf_cpu, + * so register them all after setting up CPU capacity scaling. + */ + hybrid_register_all_perf_domains(); +} + +static void hybrid_init_cpu_capacity_scaling(bool refresh) +{ + /* Bail out if enabling capacity-aware scheduling is prohibited. */ + if (no_cas) + return; + + /* + * If hybrid_max_perf_cpu is set at this point, the hybrid CPU capacity + * scaling has been enabled already and the driver is just changing the + * operation mode. + */ + if (refresh) { + hybrid_refresh_cpu_capacity_scaling(); + return; + } + + /* + * On hybrid systems, use asym capacity instead of ITMT, but because + * the capacity of SMT threads is not deterministic even approximately, + * do not do that when SMT is in use. + */ + if (hwp_is_hybrid && !sched_smt_active() && arch_enable_hybrid_capacity_scale()) { + hybrid_refresh_cpu_capacity_scaling(); + /* + * Disabling ITMT causes sched domains to be rebuilt to disable asym + * packing and enable asym capacity and EAS. + */ + sched_clear_itmt_support(); + } +} + +static bool hybrid_clear_max_perf_cpu(void) +{ + bool ret; + + guard(mutex)(&hybrid_capacity_lock); + + ret = !!hybrid_max_perf_cpu; + hybrid_max_perf_cpu = NULL; + + return ret; +} + +static void __intel_pstate_get_hwp_cap(struct cpudata *cpu) +{ + u64 cap; + + rdmsrq_on_cpu(cpu->cpu, MSR_HWP_CAPABILITIES, &cap); + WRITE_ONCE(cpu->hwp_cap_cached, cap); + cpu->pstate.max_pstate = HWP_GUARANTEED_PERF(cap); + cpu->pstate.turbo_pstate = HWP_HIGHEST_PERF(cap); +} + +static void intel_pstate_get_hwp_cap(struct cpudata *cpu) +{ + int scaling = cpu->pstate.scaling; + + __intel_pstate_get_hwp_cap(cpu); + + cpu->pstate.max_freq = cpu->pstate.max_pstate * scaling; + cpu->pstate.turbo_freq = cpu->pstate.turbo_pstate * scaling; + if (scaling != cpu->pstate.perf_ctl_scaling) { + int perf_ctl_scaling = cpu->pstate.perf_ctl_scaling; + + cpu->pstate.max_freq = rounddown(cpu->pstate.max_freq, + perf_ctl_scaling); + cpu->pstate.turbo_freq = rounddown(cpu->pstate.turbo_freq, + perf_ctl_scaling); + } +} + +static void hybrid_update_capacity(struct cpudata *cpu) +{ + unsigned int max_cap_perf; + + mutex_lock(&hybrid_capacity_lock); + + if (!hybrid_max_perf_cpu) + goto unlock; + + /* + * The maximum performance of the CPU may have changed, but assume + * that the performance of the other CPUs has not changed. + */ + max_cap_perf = hybrid_max_perf_cpu->capacity_perf; + + intel_pstate_get_hwp_cap(cpu); + + hybrid_get_capacity_perf(cpu); + /* Should hybrid_max_perf_cpu be replaced by this CPU? */ + if (cpu->capacity_perf > max_cap_perf) { + hybrid_max_perf_cpu = cpu; + hybrid_set_capacity_of_cpus(); + goto unlock; + } + + /* If this CPU is hybrid_max_perf_cpu, should it be replaced? */ + if (cpu == hybrid_max_perf_cpu && cpu->capacity_perf < max_cap_perf) { + hybrid_update_cpu_capacity_scaling(); + goto unlock; + } + + hybrid_set_cpu_capacity(cpu); + /* + * If the CPU was offline to start with and it is going online for the + * first time, a perf domain needs to be registered for it if hybrid + * capacity scaling has been enabled already. In that case, sched + * domains need to be rebuilt to take the new perf domain into account. + */ + if (hybrid_register_perf_domain(cpu->cpu)) + em_rebuild_sched_domains(); + +unlock: + mutex_unlock(&hybrid_capacity_lock); +} + +static void intel_pstate_hwp_set(unsigned int cpu) +{ + struct cpudata *cpu_data = all_cpu_data[cpu]; + int max, min; + u64 value; + s16 epp; + + max = cpu_data->max_perf_ratio; + min = cpu_data->min_perf_ratio; + + if (cpu_data->policy == CPUFREQ_POLICY_PERFORMANCE) + min = max; + + rdmsrq_on_cpu(cpu, MSR_HWP_REQUEST, &value); + + value &= ~HWP_MIN_PERF(~0L); + value |= HWP_MIN_PERF(min); + + value &= ~HWP_MAX_PERF(~0L); + value |= HWP_MAX_PERF(max); + + if (cpu_data->epp_policy == cpu_data->policy) + goto skip_epp; + + cpu_data->epp_policy = cpu_data->policy; + + if (cpu_data->policy == CPUFREQ_POLICY_PERFORMANCE) { + epp = intel_pstate_get_epp(cpu_data, value); + cpu_data->epp_powersave = epp; + /* If EPP read was failed, then don't try to write */ + if (epp < 0) + goto skip_epp; + + epp = 0; + } else { + /* skip setting EPP, when saved value is invalid */ + if (cpu_data->epp_powersave < 0) + goto skip_epp; + + /* + * No need to restore EPP when it is not zero. This + * means: + * - Policy is not changed + * - user has manually changed + * - Error reading EPB + */ + epp = intel_pstate_get_epp(cpu_data, value); + if (epp) + goto skip_epp; + + epp = cpu_data->epp_powersave; + } + if (boot_cpu_has(X86_FEATURE_HWP_EPP)) { + value &= ~GENMASK_ULL(31, 24); + value |= (u64)epp << 24; + } + +skip_epp: + WRITE_ONCE(cpu_data->hwp_req_cached, value); + wrmsrq_on_cpu(cpu, MSR_HWP_REQUEST, value); +} + +static void intel_pstate_disable_hwp_interrupt(struct cpudata *cpudata); + +static void intel_pstate_hwp_offline(struct cpudata *cpu) +{ + u64 value = READ_ONCE(cpu->hwp_req_cached); + int min_perf; + + intel_pstate_disable_hwp_interrupt(cpu); + + if (boot_cpu_has(X86_FEATURE_HWP_EPP)) { + /* + * In case the EPP has been set to "performance" by the + * active mode "performance" scaling algorithm, replace that + * temporary value with the cached EPP one. + */ + value &= ~GENMASK_ULL(31, 24); + value |= HWP_ENERGY_PERF_PREFERENCE(cpu->epp_cached); + /* + * However, make sure that EPP will be set to "performance" when + * the CPU is brought back online again and the "performance" + * scaling algorithm is still in effect. + */ + cpu->epp_policy = CPUFREQ_POLICY_UNKNOWN; + } + + /* + * Clear the desired perf field in the cached HWP request value to + * prevent nonzero desired values from being leaked into the active + * mode. + */ + value &= ~HWP_DESIRED_PERF(~0L); + WRITE_ONCE(cpu->hwp_req_cached, value); + + value &= ~GENMASK_ULL(31, 0); + min_perf = HWP_LOWEST_PERF(READ_ONCE(cpu->hwp_cap_cached)); + + /* Set hwp_max = hwp_min */ + value |= HWP_MAX_PERF(min_perf); + value |= HWP_MIN_PERF(min_perf); + + /* Set EPP to min */ + if (boot_cpu_has(X86_FEATURE_HWP_EPP)) + value |= HWP_ENERGY_PERF_PREFERENCE(HWP_EPP_POWERSAVE); + + wrmsrq_on_cpu(cpu->cpu, MSR_HWP_REQUEST, value); + + mutex_lock(&hybrid_capacity_lock); + + if (!hybrid_max_perf_cpu) { + mutex_unlock(&hybrid_capacity_lock); + + return; + } + + if (hybrid_max_perf_cpu == cpu) + hybrid_update_cpu_capacity_scaling(); + + mutex_unlock(&hybrid_capacity_lock); + + /* Reset the capacity of the CPU going offline to the initial value. */ + hybrid_clear_cpu_capacity(cpu->cpu); +} + +#define POWER_CTL_EE_ENABLE 1 +#define POWER_CTL_EE_DISABLE 2 + +/* Enable bit for Dynamic Efficiency Control (DEC) */ +#define POWER_CTL_DEC_ENABLE 27 + +static int power_ctl_ee_state; + +static void set_power_ctl_ee_state(bool input) +{ + u64 power_ctl; + + guard(mutex)(&intel_pstate_driver_lock); + + rdmsrq(MSR_IA32_POWER_CTL, power_ctl); + if (input) { + power_ctl &= ~BIT(MSR_IA32_POWER_CTL_BIT_EE); + power_ctl_ee_state = POWER_CTL_EE_ENABLE; + } else { + power_ctl |= BIT(MSR_IA32_POWER_CTL_BIT_EE); + power_ctl_ee_state = POWER_CTL_EE_DISABLE; + } + wrmsrq(MSR_IA32_POWER_CTL, power_ctl); +} + +static void intel_pstate_hwp_enable(struct cpudata *cpudata); + +static void intel_pstate_hwp_reenable(struct cpudata *cpu) +{ + intel_pstate_hwp_enable(cpu); + wrmsrq_on_cpu(cpu->cpu, MSR_HWP_REQUEST, READ_ONCE(cpu->hwp_req_cached)); +} + +static int intel_pstate_suspend(struct cpufreq_policy *policy) +{ + struct cpudata *cpu = all_cpu_data[policy->cpu]; + + pr_debug("CPU %d suspending\n", cpu->cpu); + + cpu->suspended = true; + + /* disable HWP interrupt and cancel any pending work */ + intel_pstate_disable_hwp_interrupt(cpu); + + return 0; +} + +static int intel_pstate_resume(struct cpufreq_policy *policy) +{ + struct cpudata *cpu = all_cpu_data[policy->cpu]; + + pr_debug("CPU %d resuming\n", cpu->cpu); + + /* Only restore if the system default is changed */ + if (power_ctl_ee_state == POWER_CTL_EE_ENABLE) + set_power_ctl_ee_state(true); + else if (power_ctl_ee_state == POWER_CTL_EE_DISABLE) + set_power_ctl_ee_state(false); + + if (cpu->suspended && hwp_active) { + mutex_lock(&intel_pstate_limits_lock); + + /* Re-enable HWP, because "online" has not done that. */ + intel_pstate_hwp_reenable(cpu); + + mutex_unlock(&intel_pstate_limits_lock); + } + + cpu->suspended = false; + + return 0; +} + +static void intel_pstate_update_policies(void) +{ + int cpu; + + for_each_possible_cpu(cpu) + cpufreq_update_policy(cpu); +} + +static void __intel_pstate_update_max_freq(struct cpufreq_policy *policy, + struct cpudata *cpudata) +{ + guard(cpufreq_policy_write)(policy); + + if (hwp_active) + intel_pstate_get_hwp_cap(cpudata); + + policy->cpuinfo.max_freq = READ_ONCE(global.no_turbo) ? + cpudata->pstate.max_freq : cpudata->pstate.turbo_freq; + + refresh_frequency_limits(policy); +} + +static bool intel_pstate_update_max_freq(struct cpudata *cpudata) +{ + struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpudata->cpu); + if (!policy) + return false; + + __intel_pstate_update_max_freq(policy, cpudata); + + return true; +} + +static void intel_pstate_update_limits(struct cpufreq_policy *policy) +{ + struct cpudata *cpudata = all_cpu_data[policy->cpu]; + + __intel_pstate_update_max_freq(policy, cpudata); + + hybrid_update_capacity(cpudata); +} + +static void intel_pstate_update_limits_for_all(void) +{ + int cpu; + + for_each_possible_cpu(cpu) + intel_pstate_update_max_freq(all_cpu_data[cpu]); + + mutex_lock(&hybrid_capacity_lock); + + if (hybrid_max_perf_cpu) + __hybrid_refresh_cpu_capacity_scaling(); + + mutex_unlock(&hybrid_capacity_lock); +} /************************** sysfs begin ************************/ #define show_one(file_name, object) \ static ssize_t show_##file_name \ - (struct kobject *kobj, struct attribute *attr, char *buf) \ + (struct kobject *kobj, struct kobj_attribute *attr, char *buf) \ { \ - return sprintf(buf, "%u\n", limits.object); \ + return sprintf(buf, "%u\n", global.object); \ } -static ssize_t store_no_turbo(struct kobject *a, struct attribute *b, - const char *buf, size_t count) +static ssize_t intel_pstate_show_status(char *buf); +static int intel_pstate_update_status(const char *buf, size_t size); + +static ssize_t show_status(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) { - unsigned int input; + guard(mutex)(&intel_pstate_driver_lock); + + return intel_pstate_show_status(buf); +} + +static ssize_t store_status(struct kobject *a, struct kobj_attribute *b, + const char *buf, size_t count) +{ + char *p = memchr(buf, '\n', count); int ret; - ret = sscanf(buf, "%u", &input); - if (ret != 1) + + guard(mutex)(&intel_pstate_driver_lock); + + ret = intel_pstate_update_status(buf, p ? p - buf : count); + if (ret < 0) + return ret; + + return count; +} + +static ssize_t show_turbo_pct(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct cpudata *cpu; + int total, no_turbo, turbo_pct; + uint32_t turbo_fp; + + guard(mutex)(&intel_pstate_driver_lock); + + if (!intel_pstate_driver) + return -EAGAIN; + + cpu = all_cpu_data[0]; + + total = cpu->pstate.turbo_pstate - cpu->pstate.min_pstate + 1; + no_turbo = cpu->pstate.max_pstate - cpu->pstate.min_pstate + 1; + turbo_fp = div_fp(no_turbo, total); + turbo_pct = 100 - fp_toint(mul_fp(turbo_fp, int_tofp(100))); + + return sprintf(buf, "%u\n", turbo_pct); +} + +static ssize_t show_num_pstates(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct cpudata *cpu; + int total; + + guard(mutex)(&intel_pstate_driver_lock); + + if (!intel_pstate_driver) + return -EAGAIN; + + cpu = all_cpu_data[0]; + total = cpu->pstate.turbo_pstate - cpu->pstate.min_pstate + 1; + + return sprintf(buf, "%u\n", total); +} + +static ssize_t show_no_turbo(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + guard(mutex)(&intel_pstate_driver_lock); + + if (!intel_pstate_driver) + return -EAGAIN; + + return sprintf(buf, "%u\n", global.no_turbo); +} + +static ssize_t store_no_turbo(struct kobject *a, struct kobj_attribute *b, + const char *buf, size_t count) +{ + unsigned int input; + bool no_turbo; + + if (sscanf(buf, "%u", &input) != 1) return -EINVAL; - limits.no_turbo = clamp_t(int, input, 0 , 1); + + guard(mutex)(&intel_pstate_driver_lock); + + if (!intel_pstate_driver) + return -EAGAIN; + + no_turbo = !!clamp_t(int, input, 0, 1); + + WRITE_ONCE(global.turbo_disabled, turbo_is_disabled()); + if (global.turbo_disabled && !no_turbo) { + pr_notice("Turbo disabled by BIOS or unavailable on processor\n"); + if (global.no_turbo) + return -EPERM; + + no_turbo = 1; + } + + if (no_turbo == global.no_turbo) + return count; + + WRITE_ONCE(global.no_turbo, no_turbo); + + mutex_lock(&intel_pstate_limits_lock); + + if (no_turbo) { + struct cpudata *cpu = all_cpu_data[0]; + int pct = cpu->pstate.max_pstate * 100 / cpu->pstate.turbo_pstate; + + /* Squash the global minimum into the permitted range. */ + if (global.min_perf_pct > pct) + global.min_perf_pct = pct; + } + + mutex_unlock(&intel_pstate_limits_lock); + + intel_pstate_update_limits_for_all(); + arch_set_max_freq_ratio(no_turbo); return count; } -static ssize_t store_max_perf_pct(struct kobject *a, struct attribute *b, - const char *buf, size_t count) +static void update_cpu_qos_request(int cpu, enum freq_qos_req_type type) +{ + struct cpudata *cpudata = all_cpu_data[cpu]; + unsigned int freq = cpudata->pstate.turbo_freq; + struct freq_qos_request *req; + + struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu); + if (!policy) + return; + + req = policy->driver_data; + if (!req) + return; + + if (hwp_active) + intel_pstate_get_hwp_cap(cpudata); + + if (type == FREQ_QOS_MIN) { + freq = DIV_ROUND_UP(freq * global.min_perf_pct, 100); + } else { + req++; + freq = (freq * global.max_perf_pct) / 100; + } + + if (freq_qos_update_request(req, freq) < 0) + pr_warn("Failed to update freq constraint: CPU%d\n", cpu); +} + +static void update_qos_requests(enum freq_qos_req_type type) +{ + int i; + + for_each_possible_cpu(i) + update_cpu_qos_request(i, type); +} + +static ssize_t store_max_perf_pct(struct kobject *a, struct kobj_attribute *b, + const char *buf, size_t count) { unsigned int input; int ret; + ret = sscanf(buf, "%u", &input); if (ret != 1) return -EINVAL; - limits.max_sysfs_pct = clamp_t(int, input, 0 , 100); - limits.max_perf_pct = min(limits.max_policy_pct, limits.max_sysfs_pct); - limits.max_perf = div_fp(int_tofp(limits.max_perf_pct), int_tofp(100)); + guard(mutex)(&intel_pstate_driver_lock); + + if (!intel_pstate_driver) + return -EAGAIN; + + mutex_lock(&intel_pstate_limits_lock); + + global.max_perf_pct = clamp_t(int, input, global.min_perf_pct, 100); + + mutex_unlock(&intel_pstate_limits_lock); + + if (intel_pstate_driver == &intel_pstate) + intel_pstate_update_policies(); + else + update_qos_requests(FREQ_QOS_MAX); + return count; } -static ssize_t store_min_perf_pct(struct kobject *a, struct attribute *b, - const char *buf, size_t count) +static ssize_t store_min_perf_pct(struct kobject *a, struct kobj_attribute *b, + const char *buf, size_t count) { unsigned int input; int ret; + ret = sscanf(buf, "%u", &input); if (ret != 1) return -EINVAL; - limits.min_perf_pct = clamp_t(int, input, 0 , 100); - limits.min_perf = div_fp(int_tofp(limits.min_perf_pct), int_tofp(100)); + + guard(mutex)(&intel_pstate_driver_lock); + + if (!intel_pstate_driver) + return -EAGAIN; + + mutex_lock(&intel_pstate_limits_lock); + + global.min_perf_pct = clamp_t(int, input, + min_perf_pct_min(), global.max_perf_pct); + + mutex_unlock(&intel_pstate_limits_lock); + + if (intel_pstate_driver == &intel_pstate) + intel_pstate_update_policies(); + else + update_qos_requests(FREQ_QOS_MIN); + + return count; +} + +static ssize_t show_hwp_dynamic_boost(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return sprintf(buf, "%u\n", hwp_boost); +} + +static ssize_t store_hwp_dynamic_boost(struct kobject *a, + struct kobj_attribute *b, + const char *buf, size_t count) +{ + unsigned int input; + int ret; + + ret = kstrtouint(buf, 10, &input); + if (ret) + return ret; + + guard(mutex)(&intel_pstate_driver_lock); + + hwp_boost = !!input; + intel_pstate_update_policies(); + + return count; +} + +static ssize_t show_energy_efficiency(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) +{ + u64 power_ctl; + int enable; + + rdmsrq(MSR_IA32_POWER_CTL, power_ctl); + enable = !!(power_ctl & BIT(MSR_IA32_POWER_CTL_BIT_EE)); + return sprintf(buf, "%d\n", !enable); +} + +static ssize_t store_energy_efficiency(struct kobject *a, struct kobj_attribute *b, + const char *buf, size_t count) +{ + bool input; + int ret; + + ret = kstrtobool(buf, &input); + if (ret) + return ret; + + set_power_ctl_ee_state(input); return count; } -show_one(no_turbo, no_turbo); show_one(max_perf_pct, max_perf_pct); show_one(min_perf_pct, min_perf_pct); +define_one_global_rw(status); define_one_global_rw(no_turbo); define_one_global_rw(max_perf_pct); define_one_global_rw(min_perf_pct); +define_one_global_ro(turbo_pct); +define_one_global_ro(num_pstates); +define_one_global_rw(hwp_dynamic_boost); +define_one_global_rw(energy_efficiency); static struct attribute *intel_pstate_attributes[] = { + &status.attr, &no_turbo.attr, - &max_perf_pct.attr, - &min_perf_pct.attr, NULL }; -static struct attribute_group intel_pstate_attr_group = { +static const struct attribute_group intel_pstate_attr_group = { .attrs = intel_pstate_attributes, }; + +static const struct x86_cpu_id intel_pstate_cpu_ee_disable_ids[]; + static struct kobject *intel_pstate_kobject; -static void intel_pstate_sysfs_expose_params(void) +static void __init intel_pstate_sysfs_expose_params(void) +{ + struct device *dev_root = bus_get_dev_root(&cpu_subsys); + int rc; + + if (dev_root) { + intel_pstate_kobject = kobject_create_and_add("intel_pstate", &dev_root->kobj); + put_device(dev_root); + } + if (WARN_ON(!intel_pstate_kobject)) + return; + + rc = sysfs_create_group(intel_pstate_kobject, &intel_pstate_attr_group); + if (WARN_ON(rc)) + return; + + if (!boot_cpu_has(X86_FEATURE_HYBRID_CPU)) { + rc = sysfs_create_file(intel_pstate_kobject, &turbo_pct.attr); + WARN_ON(rc); + + rc = sysfs_create_file(intel_pstate_kobject, &num_pstates.attr); + WARN_ON(rc); + } + + /* + * If per cpu limits are enforced there are no global limits, so + * return without creating max/min_perf_pct attributes + */ + if (per_cpu_limits) + return; + + rc = sysfs_create_file(intel_pstate_kobject, &max_perf_pct.attr); + WARN_ON(rc); + + rc = sysfs_create_file(intel_pstate_kobject, &min_perf_pct.attr); + WARN_ON(rc); + + if (x86_match_cpu(intel_pstate_cpu_ee_disable_ids)) { + rc = sysfs_create_file(intel_pstate_kobject, &energy_efficiency.attr); + WARN_ON(rc); + } +} + +static void __init intel_pstate_sysfs_remove(void) +{ + if (!intel_pstate_kobject) + return; + + sysfs_remove_group(intel_pstate_kobject, &intel_pstate_attr_group); + + if (!boot_cpu_has(X86_FEATURE_HYBRID_CPU)) { + sysfs_remove_file(intel_pstate_kobject, &num_pstates.attr); + sysfs_remove_file(intel_pstate_kobject, &turbo_pct.attr); + } + + if (!per_cpu_limits) { + sysfs_remove_file(intel_pstate_kobject, &max_perf_pct.attr); + sysfs_remove_file(intel_pstate_kobject, &min_perf_pct.attr); + + if (x86_match_cpu(intel_pstate_cpu_ee_disable_ids)) + sysfs_remove_file(intel_pstate_kobject, &energy_efficiency.attr); + } + + kobject_put(intel_pstate_kobject); +} + +static void intel_pstate_sysfs_expose_hwp_dynamic_boost(void) { int rc; - intel_pstate_kobject = kobject_create_and_add("intel_pstate", - &cpu_subsys.dev_root->kobj); - BUG_ON(!intel_pstate_kobject); - rc = sysfs_create_group(intel_pstate_kobject, - &intel_pstate_attr_group); - BUG_ON(rc); + if (!hwp_active) + return; + + rc = sysfs_create_file(intel_pstate_kobject, &hwp_dynamic_boost.attr); + WARN_ON_ONCE(rc); +} + +static void intel_pstate_sysfs_hide_hwp_dynamic_boost(void) +{ + if (!hwp_active) + return; + + sysfs_remove_file(intel_pstate_kobject, &hwp_dynamic_boost.attr); } /************************** sysfs end ************************/ -static int intel_pstate_min_pstate(void) +static void intel_pstate_notify_work(struct work_struct *work) +{ + struct cpudata *cpudata = + container_of(to_delayed_work(work), struct cpudata, hwp_notify_work); + + if (intel_pstate_update_max_freq(cpudata)) { + /* + * The driver will not be unregistered while this function is + * running, so update the capacity without acquiring the driver + * lock. + */ + hybrid_update_capacity(cpudata); + } + + wrmsrq_on_cpu(cpudata->cpu, MSR_HWP_STATUS, 0); +} + +static DEFINE_RAW_SPINLOCK(hwp_notify_lock); +static cpumask_t hwp_intr_enable_mask; + +#define HWP_GUARANTEED_PERF_CHANGE_STATUS BIT(0) +#define HWP_HIGHEST_PERF_CHANGE_STATUS BIT(3) + +void notify_hwp_interrupt(void) +{ + unsigned int this_cpu = smp_processor_id(); + u64 value, status_mask; + unsigned long flags; + + if (!hwp_active || !cpu_feature_enabled(X86_FEATURE_HWP_NOTIFY)) + return; + + status_mask = HWP_GUARANTEED_PERF_CHANGE_STATUS; + if (cpu_feature_enabled(X86_FEATURE_HWP_HIGHEST_PERF_CHANGE)) + status_mask |= HWP_HIGHEST_PERF_CHANGE_STATUS; + + rdmsrq_safe(MSR_HWP_STATUS, &value); + if (!(value & status_mask)) + return; + + raw_spin_lock_irqsave(&hwp_notify_lock, flags); + + if (!cpumask_test_cpu(this_cpu, &hwp_intr_enable_mask)) + goto ack_intr; + + schedule_delayed_work(&all_cpu_data[this_cpu]->hwp_notify_work, + msecs_to_jiffies(10)); + + raw_spin_unlock_irqrestore(&hwp_notify_lock, flags); + + return; + +ack_intr: + wrmsrq_safe(MSR_HWP_STATUS, 0); + raw_spin_unlock_irqrestore(&hwp_notify_lock, flags); +} + +static void intel_pstate_disable_hwp_interrupt(struct cpudata *cpudata) +{ + bool cancel_work; + + if (!cpu_feature_enabled(X86_FEATURE_HWP_NOTIFY)) + return; + + /* wrmsrq_on_cpu has to be outside spinlock as this can result in IPC */ + wrmsrq_on_cpu(cpudata->cpu, MSR_HWP_INTERRUPT, 0x00); + + raw_spin_lock_irq(&hwp_notify_lock); + cancel_work = cpumask_test_and_clear_cpu(cpudata->cpu, &hwp_intr_enable_mask); + raw_spin_unlock_irq(&hwp_notify_lock); + + if (cancel_work) + cancel_delayed_work_sync(&cpudata->hwp_notify_work); +} + +#define HWP_GUARANTEED_PERF_CHANGE_REQ BIT(0) +#define HWP_HIGHEST_PERF_CHANGE_REQ BIT(2) + +static void intel_pstate_enable_hwp_interrupt(struct cpudata *cpudata) +{ + /* Enable HWP notification interrupt for performance change */ + if (boot_cpu_has(X86_FEATURE_HWP_NOTIFY)) { + u64 interrupt_mask = HWP_GUARANTEED_PERF_CHANGE_REQ; + + raw_spin_lock_irq(&hwp_notify_lock); + INIT_DELAYED_WORK(&cpudata->hwp_notify_work, intel_pstate_notify_work); + cpumask_set_cpu(cpudata->cpu, &hwp_intr_enable_mask); + raw_spin_unlock_irq(&hwp_notify_lock); + + if (cpu_feature_enabled(X86_FEATURE_HWP_HIGHEST_PERF_CHANGE)) + interrupt_mask |= HWP_HIGHEST_PERF_CHANGE_REQ; + + /* wrmsrq_on_cpu has to be outside spinlock as this can result in IPC */ + wrmsrq_on_cpu(cpudata->cpu, MSR_HWP_INTERRUPT, interrupt_mask); + wrmsrq_on_cpu(cpudata->cpu, MSR_HWP_STATUS, 0); + } +} + +static void intel_pstate_update_epp_defaults(struct cpudata *cpudata) +{ + cpudata->epp_default = intel_pstate_get_epp(cpudata, 0); + + /* + * If the EPP is set by firmware, which means that firmware enabled HWP + * - Is equal or less than 0x80 (default balance_perf EPP) + * - But less performance oriented than performance EPP + * then use this as new balance_perf EPP. + */ + if (hwp_forced && cpudata->epp_default <= HWP_EPP_BALANCE_PERFORMANCE && + cpudata->epp_default > HWP_EPP_PERFORMANCE) { + epp_values[EPP_INDEX_BALANCE_PERFORMANCE] = cpudata->epp_default; + return; + } + + /* + * If this CPU gen doesn't call for change in balance_perf + * EPP return. + */ + if (epp_values[EPP_INDEX_BALANCE_PERFORMANCE] == HWP_EPP_BALANCE_PERFORMANCE) + return; + + /* + * Use hard coded value per gen to update the balance_perf + * and default EPP. + */ + cpudata->epp_default = epp_values[EPP_INDEX_BALANCE_PERFORMANCE]; + intel_pstate_set_epp(cpudata, cpudata->epp_default); +} + +static void intel_pstate_hwp_enable(struct cpudata *cpudata) +{ + /* First disable HWP notification interrupt till we activate again */ + if (boot_cpu_has(X86_FEATURE_HWP_NOTIFY)) + wrmsrq_on_cpu(cpudata->cpu, MSR_HWP_INTERRUPT, 0x00); + + wrmsrq_on_cpu(cpudata->cpu, MSR_PM_ENABLE, 0x1); + + intel_pstate_enable_hwp_interrupt(cpudata); + + if (cpudata->epp_default >= 0) + return; + + intel_pstate_update_epp_defaults(cpudata); +} + +static u64 get_perf_ctl_val(int pstate) +{ + u64 val; + + val = (u64)pstate << 8; + if (READ_ONCE(global.no_turbo) && !READ_ONCE(global.turbo_disabled) && + cpu_feature_enabled(X86_FEATURE_IDA)) + val |= (u64)1 << 32; + + return val; +} + +static int atom_get_min_pstate(int not_used) +{ + u64 value; + + rdmsrq(MSR_ATOM_CORE_RATIOS, value); + return (value >> 8) & 0x7F; +} + +static int atom_get_max_pstate(int not_used) +{ + u64 value; + + rdmsrq(MSR_ATOM_CORE_RATIOS, value); + return (value >> 16) & 0x7F; +} + +static int atom_get_turbo_pstate(int not_used) { u64 value; - rdmsrl(MSR_PLATFORM_INFO, value); + + rdmsrq(MSR_ATOM_CORE_TURBO_RATIOS, value); + return value & 0x7F; +} + +static u64 atom_get_val(struct cpudata *cpudata, int pstate) +{ + u64 val = get_perf_ctl_val(pstate); + int32_t vid_fp; + u32 vid; + + vid_fp = cpudata->vid.min + mul_fp( + int_tofp(pstate - cpudata->pstate.min_pstate), + cpudata->vid.ratio); + + vid_fp = clamp_t(int32_t, vid_fp, cpudata->vid.min, cpudata->vid.max); + vid = ceiling_fp(vid_fp); + + if (pstate > cpudata->pstate.max_pstate) + vid = cpudata->vid.turbo; + + return val | vid; +} + +static int silvermont_get_scaling(void) +{ + u64 value; + int i; + /* Defined in Table 35-6 from SDM (Sept 2015) */ + static int silvermont_freq_table[] = { + 83300, 100000, 133300, 116700, 80000}; + + rdmsrq(MSR_FSB_FREQ, value); + i = value & 0x7; + WARN_ON(i > 4); + + return silvermont_freq_table[i]; +} + +static int airmont_get_scaling(void) +{ + u64 value; + int i; + /* Defined in Table 35-10 from SDM (Sept 2015) */ + static int airmont_freq_table[] = { + 83300, 100000, 133300, 116700, 80000, + 93300, 90000, 88900, 87500}; + + rdmsrq(MSR_FSB_FREQ, value); + i = value & 0xF; + WARN_ON(i > 8); + + return airmont_freq_table[i]; +} + +static void atom_get_vid(struct cpudata *cpudata) +{ + u64 value; + + rdmsrq(MSR_ATOM_CORE_VIDS, value); + cpudata->vid.min = int_tofp((value >> 8) & 0x7f); + cpudata->vid.max = int_tofp((value >> 16) & 0x7f); + cpudata->vid.ratio = div_fp( + cpudata->vid.max - cpudata->vid.min, + int_tofp(cpudata->pstate.max_pstate - + cpudata->pstate.min_pstate)); + + rdmsrq(MSR_ATOM_CORE_TURBO_VIDS, value); + cpudata->vid.turbo = value & 0x7f; +} + +static int core_get_min_pstate(int cpu) +{ + u64 value; + + rdmsrq_on_cpu(cpu, MSR_PLATFORM_INFO, &value); return (value >> 40) & 0xFF; } -static int intel_pstate_max_pstate(void) +static int core_get_max_pstate_physical(int cpu) { u64 value; - rdmsrl(MSR_PLATFORM_INFO, value); + + rdmsrq_on_cpu(cpu, MSR_PLATFORM_INFO, &value); return (value >> 8) & 0xFF; } -static int intel_pstate_turbo_pstate(void) +static int core_get_tdp_ratio(int cpu, u64 plat_info) +{ + /* Check how many TDP levels present */ + if (plat_info & 0x600000000) { + u64 tdp_ctrl; + u64 tdp_ratio; + int tdp_msr; + int err; + + /* Get the TDP level (0, 1, 2) to get ratios */ + err = rdmsrq_safe_on_cpu(cpu, MSR_CONFIG_TDP_CONTROL, &tdp_ctrl); + if (err) + return err; + + /* TDP MSR are continuous starting at 0x648 */ + tdp_msr = MSR_CONFIG_TDP_NOMINAL + (tdp_ctrl & 0x03); + err = rdmsrq_safe_on_cpu(cpu, tdp_msr, &tdp_ratio); + if (err) + return err; + + /* For level 1 and 2, bits[23:16] contain the ratio */ + if (tdp_ctrl & 0x03) + tdp_ratio >>= 16; + + tdp_ratio &= 0xff; /* ratios are only 8 bits long */ + pr_debug("tdp_ratio %x\n", (int)tdp_ratio); + + return (int)tdp_ratio; + } + + return -ENXIO; +} + +static int core_get_max_pstate(int cpu) +{ + u64 tar; + u64 plat_info; + int max_pstate; + int tdp_ratio; + int err; + + rdmsrq_on_cpu(cpu, MSR_PLATFORM_INFO, &plat_info); + max_pstate = (plat_info >> 8) & 0xFF; + + tdp_ratio = core_get_tdp_ratio(cpu, plat_info); + if (tdp_ratio <= 0) + return max_pstate; + + if (hwp_active) { + /* Turbo activation ratio is not used on HWP platforms */ + return tdp_ratio; + } + + err = rdmsrq_safe_on_cpu(cpu, MSR_TURBO_ACTIVATION_RATIO, &tar); + if (!err) { + int tar_levels; + + /* Do some sanity checking for safety */ + tar_levels = tar & 0xff; + if (tdp_ratio - 1 == tar_levels) { + max_pstate = tar_levels; + pr_debug("max_pstate=TAC %x\n", max_pstate); + } + } + + return max_pstate; +} + +static int core_get_turbo_pstate(int cpu) { u64 value; int nont, ret; - rdmsrl(MSR_NHM_TURBO_RATIO_LIMIT, value); - nont = intel_pstate_max_pstate(); - ret = ((value) & 255); + + rdmsrq_on_cpu(cpu, MSR_TURBO_RATIO_LIMIT, &value); + nont = core_get_max_pstate(cpu); + ret = (value) & 255; if (ret <= nont) ret = nont; return ret; } -static void intel_pstate_get_min_max(struct cpudata *cpu, int *min, int *max) +static u64 core_get_val(struct cpudata *cpudata, int pstate) { - int max_perf = cpu->pstate.turbo_pstate; - int min_perf; - if (limits.no_turbo) - max_perf = cpu->pstate.max_pstate; + return get_perf_ctl_val(pstate); +} - max_perf = fp_toint(mul_fp(int_tofp(max_perf), limits.max_perf)); - *max = clamp_t(int, max_perf, - cpu->pstate.min_pstate, cpu->pstate.turbo_pstate); +static int knl_get_aperf_mperf_shift(void) +{ + return 10; +} + +static int knl_get_turbo_pstate(int cpu) +{ + u64 value; + int nont, ret; - min_perf = fp_toint(mul_fp(int_tofp(max_perf), limits.min_perf)); - *min = clamp_t(int, min_perf, - cpu->pstate.min_pstate, max_perf); + rdmsrq_on_cpu(cpu, MSR_TURBO_RATIO_LIMIT, &value); + nont = core_get_max_pstate(cpu); + ret = (((value) >> 8) & 0xFF); + if (ret <= nont) + ret = nont; + return ret; +} + +static int hwp_get_cpu_scaling(int cpu) +{ + if (hybrid_scaling_factor) { + /* + * Return the hybrid scaling factor for P-cores and use the + * default core scaling for E-cores. + */ + if (hybrid_get_cpu_type(cpu) == INTEL_CPU_TYPE_CORE) + return hybrid_scaling_factor; + + return core_get_scaling(); + } + + /* Use core scaling on non-hybrid systems. */ + if (!cpu_feature_enabled(X86_FEATURE_HYBRID_CPU)) + return core_get_scaling(); + + /* + * The system is hybrid, but the hybrid scaling factor is not known or + * the CPU type is not one of the above, so use CPPC to compute the + * scaling factor for this CPU. + */ + return intel_pstate_cppc_get_scaling(cpu); } static void intel_pstate_set_pstate(struct cpudata *cpu, int pstate) { - int max_perf, min_perf; + trace_cpu_frequency(pstate * cpu->pstate.scaling, cpu->cpu); + cpu->pstate.current_pstate = pstate; + /* + * Generally, there is no guarantee that this code will always run on + * the CPU being updated, so force the register update to run on the + * right CPU. + */ + wrmsrq_on_cpu(cpu->cpu, MSR_IA32_PERF_CTL, + pstate_funcs.get_val(cpu, pstate)); +} + +static void intel_pstate_set_min_pstate(struct cpudata *cpu) +{ + intel_pstate_set_pstate(cpu, cpu->pstate.min_pstate); +} - intel_pstate_get_min_max(cpu, &min_perf, &max_perf); +static void intel_pstate_get_cpu_pstates(struct cpudata *cpu) +{ + int perf_ctl_scaling = pstate_funcs.get_scaling(); - pstate = clamp_t(int, pstate, min_perf, max_perf); + cpu->pstate.max_pstate_physical = pstate_funcs.get_max_physical(cpu->cpu); + cpu->pstate.min_pstate = pstate_funcs.get_min(cpu->cpu); + cpu->pstate.perf_ctl_scaling = perf_ctl_scaling; - if (pstate == cpu->pstate.current_pstate) - return; + if (hwp_active && !hwp_mode_bdw) { + __intel_pstate_get_hwp_cap(cpu); - trace_cpu_frequency(pstate * 100000, cpu->cpu); + if (pstate_funcs.get_cpu_scaling) { + cpu->pstate.scaling = pstate_funcs.get_cpu_scaling(cpu->cpu); + intel_pstate_hybrid_hwp_adjust(cpu); + } else { + cpu->pstate.scaling = perf_ctl_scaling; + } + /* + * If the CPU is going online for the first time and it was + * offline initially, asym capacity scaling needs to be updated. + */ + hybrid_update_capacity(cpu); + } else { + cpu->pstate.scaling = perf_ctl_scaling; + cpu->pstate.max_pstate = pstate_funcs.get_max(cpu->cpu); + cpu->pstate.turbo_pstate = pstate_funcs.get_turbo(cpu->cpu); + } - cpu->pstate.current_pstate = pstate; - wrmsrl(MSR_IA32_PERF_CTL, pstate << 8); + if (cpu->pstate.scaling == perf_ctl_scaling) { + cpu->pstate.min_freq = cpu->pstate.min_pstate * perf_ctl_scaling; + cpu->pstate.max_freq = cpu->pstate.max_pstate * perf_ctl_scaling; + cpu->pstate.turbo_freq = cpu->pstate.turbo_pstate * perf_ctl_scaling; + } + + if (pstate_funcs.get_aperf_mperf_shift) + cpu->aperf_mperf_shift = pstate_funcs.get_aperf_mperf_shift(); + if (pstate_funcs.get_vid) + pstate_funcs.get_vid(cpu); + + intel_pstate_set_min_pstate(cpu); } -static inline void intel_pstate_pstate_increase(struct cpudata *cpu, int steps) +/* + * Long hold time will keep high perf limits for long time, + * which negatively impacts perf/watt for some workloads, + * like specpower. 3ms is based on experiements on some + * workoads. + */ +static int hwp_boost_hold_time_ns = 3 * NSEC_PER_MSEC; + +static inline void intel_pstate_hwp_boost_up(struct cpudata *cpu) { - int target; - target = cpu->pstate.current_pstate + steps; + u64 hwp_req = READ_ONCE(cpu->hwp_req_cached); + u64 hwp_cap = READ_ONCE(cpu->hwp_cap_cached); + u32 max_limit = (hwp_req & 0xff00) >> 8; + u32 min_limit = (hwp_req & 0xff); + u32 boost_level1; - intel_pstate_set_pstate(cpu, target); + /* + * Cases to consider (User changes via sysfs or boot time): + * If, P0 (Turbo max) = P1 (Guaranteed max) = min: + * No boost, return. + * If, P0 (Turbo max) > P1 (Guaranteed max) = min: + * Should result in one level boost only for P0. + * If, P0 (Turbo max) = P1 (Guaranteed max) > min: + * Should result in two level boost: + * (min + p1)/2 and P1. + * If, P0 (Turbo max) > P1 (Guaranteed max) > min: + * Should result in three level boost: + * (min + p1)/2, P1 and P0. + */ + + /* If max and min are equal or already at max, nothing to boost */ + if (max_limit == min_limit || cpu->hwp_boost_min >= max_limit) + return; + + if (!cpu->hwp_boost_min) + cpu->hwp_boost_min = min_limit; + + /* level at half way mark between min and guranteed */ + boost_level1 = (HWP_GUARANTEED_PERF(hwp_cap) + min_limit) >> 1; + + if (cpu->hwp_boost_min < boost_level1) + cpu->hwp_boost_min = boost_level1; + else if (cpu->hwp_boost_min < HWP_GUARANTEED_PERF(hwp_cap)) + cpu->hwp_boost_min = HWP_GUARANTEED_PERF(hwp_cap); + else if (cpu->hwp_boost_min == HWP_GUARANTEED_PERF(hwp_cap) && + max_limit != HWP_GUARANTEED_PERF(hwp_cap)) + cpu->hwp_boost_min = max_limit; + else + return; + + hwp_req = (hwp_req & ~GENMASK_ULL(7, 0)) | cpu->hwp_boost_min; + wrmsrq(MSR_HWP_REQUEST, hwp_req); + cpu->last_update = cpu->sample.time; } -static inline void intel_pstate_pstate_decrease(struct cpudata *cpu, int steps) +static inline void intel_pstate_hwp_boost_down(struct cpudata *cpu) { - int target; - target = cpu->pstate.current_pstate - steps; - intel_pstate_set_pstate(cpu, target); + if (cpu->hwp_boost_min) { + bool expired; + + /* Check if we are idle for hold time to boost down */ + expired = time_after64(cpu->sample.time, cpu->last_update + + hwp_boost_hold_time_ns); + if (expired) { + wrmsrq(MSR_HWP_REQUEST, cpu->hwp_req_cached); + cpu->hwp_boost_min = 0; + } + } + cpu->last_update = cpu->sample.time; } -static void intel_pstate_get_cpu_pstates(struct cpudata *cpu) +static inline void intel_pstate_update_util_hwp_local(struct cpudata *cpu, + u64 time) { - sprintf(cpu->name, "Intel 2nd generation core"); + cpu->sample.time = time; - cpu->pstate.min_pstate = intel_pstate_min_pstate(); - cpu->pstate.max_pstate = intel_pstate_max_pstate(); - cpu->pstate.turbo_pstate = intel_pstate_turbo_pstate(); + if (cpu->sched_flags & SCHED_CPUFREQ_IOWAIT) { + bool do_io = false; - /* - * goto max pstate so we don't slow up boot if we are built-in if we are - * a module we will take care of it during normal operation - */ - intel_pstate_set_pstate(cpu, cpu->pstate.max_pstate); + cpu->sched_flags = 0; + /* + * Set iowait_boost flag and update time. Since IO WAIT flag + * is set all the time, we can't just conclude that there is + * some IO bound activity is scheduled on this CPU with just + * one occurrence. If we receive at least two in two + * consecutive ticks, then we treat as boost candidate. + */ + if (time_before64(time, cpu->last_io_update + 2 * TICK_NSEC)) + do_io = true; + + cpu->last_io_update = time; + + if (do_io) + intel_pstate_hwp_boost_up(cpu); + + } else { + intel_pstate_hwp_boost_down(cpu); + } } -static inline void intel_pstate_calc_busy(struct cpudata *cpu, - struct sample *sample) +static inline void intel_pstate_update_util_hwp(struct update_util_data *data, + u64 time, unsigned int flags) { - u64 core_pct; - core_pct = div64_u64(sample->aperf * 100, sample->mperf); - sample->freq = cpu->pstate.max_pstate * core_pct * 1000; + struct cpudata *cpu = container_of(data, struct cpudata, update_util); - sample->core_pct_busy = core_pct; + cpu->sched_flags |= flags; + + if (smp_processor_id() == cpu->cpu) + intel_pstate_update_util_hwp_local(cpu, time); } -static inline void intel_pstate_sample(struct cpudata *cpu) +static inline void intel_pstate_calc_avg_perf(struct cpudata *cpu) { - u64 aperf, mperf; + struct sample *sample = &cpu->sample; - rdmsrl(MSR_IA32_APERF, aperf); - rdmsrl(MSR_IA32_MPERF, mperf); - cpu->sample_ptr = (cpu->sample_ptr + 1) % SAMPLE_COUNT; - cpu->samples[cpu->sample_ptr].aperf = aperf; - cpu->samples[cpu->sample_ptr].mperf = mperf; - cpu->samples[cpu->sample_ptr].aperf -= cpu->prev_aperf; - cpu->samples[cpu->sample_ptr].mperf -= cpu->prev_mperf; + sample->core_avg_perf = div_ext_fp(sample->aperf, sample->mperf); +} + +static inline bool intel_pstate_sample(struct cpudata *cpu, u64 time) +{ + u64 aperf, mperf; + unsigned long flags; + u64 tsc; + + local_irq_save(flags); + rdmsrq(MSR_IA32_APERF, aperf); + rdmsrq(MSR_IA32_MPERF, mperf); + tsc = rdtsc(); + if (cpu->prev_mperf == mperf || cpu->prev_tsc == tsc) { + local_irq_restore(flags); + return false; + } + local_irq_restore(flags); - intel_pstate_calc_busy(cpu, &cpu->samples[cpu->sample_ptr]); + cpu->last_sample_time = cpu->sample.time; + cpu->sample.time = time; + cpu->sample.aperf = aperf; + cpu->sample.mperf = mperf; + cpu->sample.tsc = tsc; + cpu->sample.aperf -= cpu->prev_aperf; + cpu->sample.mperf -= cpu->prev_mperf; + cpu->sample.tsc -= cpu->prev_tsc; cpu->prev_aperf = aperf; cpu->prev_mperf = mperf; + cpu->prev_tsc = tsc; + /* + * First time this function is invoked in a given cycle, all of the + * previous sample data fields are equal to zero or stale and they must + * be populated with meaningful numbers for things to work, so assume + * that sample.time will always be reset before setting the utilization + * update hook and make the caller skip the sample then. + */ + if (likely(cpu->last_sample_time)) { + intel_pstate_calc_avg_perf(cpu); + return true; + } + return false; } -static inline void intel_pstate_set_sample_time(struct cpudata *cpu) +static inline int32_t get_avg_frequency(struct cpudata *cpu) { - int sample_time, delay; + return mul_ext_fp(cpu->sample.core_avg_perf, cpu_khz); +} - sample_time = cpu->pstate_policy->sample_rate_ms; - delay = msecs_to_jiffies(sample_time); - mod_timer_pinned(&cpu->timer, jiffies + delay); +static inline int32_t get_avg_pstate(struct cpudata *cpu) +{ + return mul_ext_fp(cpu->pstate.max_pstate_physical, + cpu->sample.core_avg_perf); } -static inline int intel_pstate_get_scaled_busy(struct cpudata *cpu) +static inline int32_t get_target_pstate(struct cpudata *cpu) { - int32_t busy_scaled; - int32_t core_busy, turbo_pstate, current_pstate; + struct sample *sample = &cpu->sample; + int32_t busy_frac; + int target, avg_pstate; + + busy_frac = div_fp(sample->mperf << cpu->aperf_mperf_shift, + sample->tsc); + + if (busy_frac < cpu->iowait_boost) + busy_frac = cpu->iowait_boost; - core_busy = int_tofp(cpu->samples[cpu->sample_ptr].core_pct_busy); - turbo_pstate = int_tofp(cpu->pstate.turbo_pstate); - current_pstate = int_tofp(cpu->pstate.current_pstate); - busy_scaled = mul_fp(core_busy, div_fp(turbo_pstate, current_pstate)); + sample->busy_scaled = busy_frac * 100; + + target = READ_ONCE(global.no_turbo) ? + cpu->pstate.max_pstate : cpu->pstate.turbo_pstate; + target += target >> 2; + target = mul_fp(target, busy_frac); + if (target < cpu->pstate.min_pstate) + target = cpu->pstate.min_pstate; + + /* + * If the average P-state during the previous cycle was higher than the + * current target, add 50% of the difference to the target to reduce + * possible performance oscillations and offset possible performance + * loss related to moving the workload from one CPU to another within + * a package/module. + */ + avg_pstate = get_avg_pstate(cpu); + if (avg_pstate > target) + target += (avg_pstate - target) >> 1; - return fp_toint(busy_scaled); + return target; } -static inline void intel_pstate_adjust_busy_pstate(struct cpudata *cpu) +static int intel_pstate_prepare_request(struct cpudata *cpu, int pstate) { - int busy_scaled; - struct _pid *pid; - signed int ctl = 0; - int steps; + int min_pstate = max(cpu->pstate.min_pstate, cpu->min_perf_ratio); + int max_pstate = max(min_pstate, cpu->max_perf_ratio); - pid = &cpu->pid; - busy_scaled = intel_pstate_get_scaled_busy(cpu); + return clamp_t(int, pstate, min_pstate, max_pstate); +} - ctl = pid_calc(pid, busy_scaled); +static void intel_pstate_update_pstate(struct cpudata *cpu, int pstate) +{ + if (pstate == cpu->pstate.current_pstate) + return; - steps = abs(ctl); - if (ctl < 0) - intel_pstate_pstate_increase(cpu, steps); - else - intel_pstate_pstate_decrease(cpu, steps); + cpu->pstate.current_pstate = pstate; + wrmsrq(MSR_IA32_PERF_CTL, pstate_funcs.get_val(cpu, pstate)); } -static void intel_pstate_timer_func(unsigned long __data) +static void intel_pstate_adjust_pstate(struct cpudata *cpu) { - struct cpudata *cpu = (struct cpudata *) __data; + int from = cpu->pstate.current_pstate; + struct sample *sample; + int target_pstate; + + target_pstate = get_target_pstate(cpu); + target_pstate = intel_pstate_prepare_request(cpu, target_pstate); + trace_cpu_frequency(target_pstate * cpu->pstate.scaling, cpu->cpu); + intel_pstate_update_pstate(cpu, target_pstate); + + sample = &cpu->sample; + trace_pstate_sample(mul_ext_fp(100, sample->core_avg_perf), + fp_toint(sample->busy_scaled), + from, + cpu->pstate.current_pstate, + sample->mperf, + sample->aperf, + sample->tsc, + get_avg_frequency(cpu), + fp_toint(cpu->iowait_boost * 100)); +} - intel_pstate_sample(cpu); - intel_pstate_adjust_busy_pstate(cpu); +static void intel_pstate_update_util(struct update_util_data *data, u64 time, + unsigned int flags) +{ + struct cpudata *cpu = container_of(data, struct cpudata, update_util); + u64 delta_ns; - if (cpu->pstate.current_pstate == cpu->pstate.min_pstate) { - cpu->min_pstate_count++; - if (!(cpu->min_pstate_count % 5)) { - intel_pstate_set_pstate(cpu, cpu->pstate.max_pstate); + /* Don't allow remote callbacks */ + if (smp_processor_id() != cpu->cpu) + return; + + delta_ns = time - cpu->last_update; + if (flags & SCHED_CPUFREQ_IOWAIT) { + /* Start over if the CPU may have been idle. */ + if (delta_ns > TICK_NSEC) { + cpu->iowait_boost = ONE_EIGHTH_FP; + } else if (cpu->iowait_boost >= ONE_EIGHTH_FP) { + cpu->iowait_boost <<= 1; + if (cpu->iowait_boost > int_tofp(1)) + cpu->iowait_boost = int_tofp(1); + } else { + cpu->iowait_boost = ONE_EIGHTH_FP; } - } else - cpu->min_pstate_count = 0; + } else if (cpu->iowait_boost) { + /* Clear iowait_boost if the CPU may have been idle. */ + if (delta_ns > TICK_NSEC) + cpu->iowait_boost = 0; + else + cpu->iowait_boost >>= 1; + } + cpu->last_update = time; + delta_ns = time - cpu->sample.time; + if ((s64)delta_ns < INTEL_PSTATE_SAMPLING_INTERVAL) + return; - intel_pstate_set_sample_time(cpu); + if (intel_pstate_sample(cpu, time)) + intel_pstate_adjust_pstate(cpu); } -#define ICPU(model, policy) \ - { X86_VENDOR_INTEL, 6, model, X86_FEATURE_ANY, (unsigned long)&policy } +static struct pstate_funcs core_funcs = { + .get_max = core_get_max_pstate, + .get_max_physical = core_get_max_pstate_physical, + .get_min = core_get_min_pstate, + .get_turbo = core_get_turbo_pstate, + .get_scaling = core_get_scaling, + .get_val = core_get_val, +}; + +static const struct pstate_funcs silvermont_funcs = { + .get_max = atom_get_max_pstate, + .get_max_physical = atom_get_max_pstate, + .get_min = atom_get_min_pstate, + .get_turbo = atom_get_turbo_pstate, + .get_val = atom_get_val, + .get_scaling = silvermont_get_scaling, + .get_vid = atom_get_vid, +}; + +static const struct pstate_funcs airmont_funcs = { + .get_max = atom_get_max_pstate, + .get_max_physical = atom_get_max_pstate, + .get_min = atom_get_min_pstate, + .get_turbo = atom_get_turbo_pstate, + .get_val = atom_get_val, + .get_scaling = airmont_get_scaling, + .get_vid = atom_get_vid, +}; + +static const struct pstate_funcs knl_funcs = { + .get_max = core_get_max_pstate, + .get_max_physical = core_get_max_pstate_physical, + .get_min = core_get_min_pstate, + .get_turbo = knl_get_turbo_pstate, + .get_aperf_mperf_shift = knl_get_aperf_mperf_shift, + .get_scaling = core_get_scaling, + .get_val = core_get_val, +}; + +#define X86_MATCH(vfm, policy) \ + X86_MATCH_VFM_FEATURE(vfm, X86_FEATURE_APERFMPERF, &policy) static const struct x86_cpu_id intel_pstate_cpu_ids[] = { - ICPU(0x2a, default_policy), - ICPU(0x2d, default_policy), - ICPU(0x3a, default_policy), + X86_MATCH(INTEL_SANDYBRIDGE, core_funcs), + X86_MATCH(INTEL_SANDYBRIDGE_X, core_funcs), + X86_MATCH(INTEL_ATOM_SILVERMONT, silvermont_funcs), + X86_MATCH(INTEL_IVYBRIDGE, core_funcs), + X86_MATCH(INTEL_HASWELL, core_funcs), + X86_MATCH(INTEL_BROADWELL, core_funcs), + X86_MATCH(INTEL_IVYBRIDGE_X, core_funcs), + X86_MATCH(INTEL_HASWELL_X, core_funcs), + X86_MATCH(INTEL_HASWELL_L, core_funcs), + X86_MATCH(INTEL_HASWELL_G, core_funcs), + X86_MATCH(INTEL_BROADWELL_G, core_funcs), + X86_MATCH(INTEL_ATOM_AIRMONT, airmont_funcs), + X86_MATCH(INTEL_SKYLAKE_L, core_funcs), + X86_MATCH(INTEL_BROADWELL_X, core_funcs), + X86_MATCH(INTEL_SKYLAKE, core_funcs), + X86_MATCH(INTEL_BROADWELL_D, core_funcs), + X86_MATCH(INTEL_XEON_PHI_KNL, knl_funcs), + X86_MATCH(INTEL_XEON_PHI_KNM, knl_funcs), + X86_MATCH(INTEL_ATOM_GOLDMONT, core_funcs), + X86_MATCH(INTEL_ATOM_GOLDMONT_PLUS, core_funcs), + X86_MATCH(INTEL_SKYLAKE_X, core_funcs), + X86_MATCH(INTEL_COMETLAKE, core_funcs), + X86_MATCH(INTEL_ICELAKE_X, core_funcs), + X86_MATCH(INTEL_TIGERLAKE, core_funcs), + X86_MATCH(INTEL_SAPPHIRERAPIDS_X, core_funcs), + X86_MATCH(INTEL_EMERALDRAPIDS_X, core_funcs), + X86_MATCH(INTEL_GRANITERAPIDS_D, core_funcs), + X86_MATCH(INTEL_GRANITERAPIDS_X, core_funcs), {} }; MODULE_DEVICE_TABLE(x86cpu, intel_pstate_cpu_ids); +#ifdef CONFIG_ACPI +static const struct x86_cpu_id intel_pstate_cpu_oob_ids[] __initconst = { + X86_MATCH(INTEL_BROADWELL_D, core_funcs), + X86_MATCH(INTEL_BROADWELL_X, core_funcs), + X86_MATCH(INTEL_SKYLAKE_X, core_funcs), + X86_MATCH(INTEL_ICELAKE_X, core_funcs), + X86_MATCH(INTEL_SAPPHIRERAPIDS_X, core_funcs), + X86_MATCH(INTEL_EMERALDRAPIDS_X, core_funcs), + X86_MATCH(INTEL_GRANITERAPIDS_D, core_funcs), + X86_MATCH(INTEL_GRANITERAPIDS_X, core_funcs), + X86_MATCH(INTEL_ATOM_CRESTMONT, core_funcs), + X86_MATCH(INTEL_ATOM_CRESTMONT_X, core_funcs), + X86_MATCH(INTEL_ATOM_DARKMONT_X, core_funcs), + X86_MATCH(INTEL_DIAMONDRAPIDS_X, core_funcs), + {} +}; +#endif + +static const struct x86_cpu_id intel_pstate_cpu_ee_disable_ids[] = { + X86_MATCH(INTEL_KABYLAKE, core_funcs), + {} +}; + static int intel_pstate_init_cpu(unsigned int cpunum) { - - const struct x86_cpu_id *id; struct cpudata *cpu; - id = x86_match_cpu(intel_pstate_cpu_ids); - if (!id) - return -ENODEV; + cpu = all_cpu_data[cpunum]; - all_cpu_data[cpunum] = kzalloc(sizeof(struct cpudata), GFP_KERNEL); - if (!all_cpu_data[cpunum]) - return -ENOMEM; + if (!cpu) { + cpu = kzalloc(sizeof(*cpu), GFP_KERNEL); + if (!cpu) + return -ENOMEM; - cpu = all_cpu_data[cpunum]; + WRITE_ONCE(all_cpu_data[cpunum], cpu); - intel_pstate_get_cpu_pstates(cpu); + cpu->cpu = cpunum; + + cpu->epp_default = -EINVAL; + + if (hwp_active) { + intel_pstate_hwp_enable(cpu); + + if (intel_pstate_acpi_pm_profile_server()) + hwp_boost = true; + } + } else if (hwp_active) { + /* + * Re-enable HWP in case this happens after a resume from ACPI + * S3 if the CPU was offline during the whole system/resume + * cycle. + */ + intel_pstate_hwp_reenable(cpu); + } - cpu->cpu = cpunum; - cpu->pstate_policy = - (struct pstate_adjust_policy *)id->driver_data; - init_timer_deferrable(&cpu->timer); - cpu->timer.function = intel_pstate_timer_func; - cpu->timer.data = - (unsigned long)cpu; - cpu->timer.expires = jiffies + HZ/100; - intel_pstate_busy_pid_reset(cpu); - intel_pstate_sample(cpu); - intel_pstate_set_pstate(cpu, cpu->pstate.max_pstate); + cpu->epp_powersave = -EINVAL; + cpu->epp_policy = CPUFREQ_POLICY_UNKNOWN; - add_timer_on(&cpu->timer, cpunum); + intel_pstate_get_cpu_pstates(cpu); - pr_info("Intel pstate controlling: cpu %d\n", cpunum); + pr_debug("controlling: cpu %d\n", cpunum); return 0; } -static unsigned int intel_pstate_get(unsigned int cpu_num) +static void intel_pstate_set_update_util_hook(unsigned int cpu_num) { - struct sample *sample; - struct cpudata *cpu; + struct cpudata *cpu = all_cpu_data[cpu_num]; - cpu = all_cpu_data[cpu_num]; - if (!cpu) - return 0; - sample = &cpu->samples[cpu->sample_ptr]; - return sample->freq; + if (hwp_active && !hwp_boost) + return; + + if (cpu->update_util_set) + return; + + /* Prevent intel_pstate_update_util() from using stale data. */ + cpu->sample.time = 0; + cpufreq_add_update_util_hook(cpu_num, &cpu->update_util, + (hwp_active ? + intel_pstate_update_util_hwp : + intel_pstate_update_util)); + cpu->update_util_set = true; +} + +static void intel_pstate_clear_update_util_hook(unsigned int cpu) +{ + struct cpudata *cpu_data = all_cpu_data[cpu]; + + if (!cpu_data->update_util_set) + return; + + cpufreq_remove_update_util_hook(cpu); + cpu_data->update_util_set = false; + synchronize_rcu(); +} + +static int intel_pstate_get_max_freq(struct cpudata *cpu) +{ + return READ_ONCE(global.no_turbo) ? + cpu->pstate.max_freq : cpu->pstate.turbo_freq; +} + +static void intel_pstate_update_perf_limits(struct cpudata *cpu, + unsigned int policy_min, + unsigned int policy_max) +{ + int perf_ctl_scaling = cpu->pstate.perf_ctl_scaling; + int32_t max_policy_perf, min_policy_perf; + + max_policy_perf = policy_max / perf_ctl_scaling; + if (policy_max == policy_min) { + min_policy_perf = max_policy_perf; + } else { + min_policy_perf = policy_min / perf_ctl_scaling; + min_policy_perf = clamp_t(int32_t, min_policy_perf, + 0, max_policy_perf); + } + + /* + * HWP needs some special consideration, because HWP_REQUEST uses + * abstract values to represent performance rather than pure ratios. + */ + if (hwp_active && cpu->pstate.scaling != perf_ctl_scaling) { + int freq; + + freq = max_policy_perf * perf_ctl_scaling; + max_policy_perf = intel_pstate_freq_to_hwp(cpu, freq); + freq = min_policy_perf * perf_ctl_scaling; + min_policy_perf = intel_pstate_freq_to_hwp(cpu, freq); + } + + pr_debug("cpu:%d min_policy_perf:%d max_policy_perf:%d\n", + cpu->cpu, min_policy_perf, max_policy_perf); + + /* Normalize user input to [min_perf, max_perf] */ + if (per_cpu_limits) { + cpu->min_perf_ratio = min_policy_perf; + cpu->max_perf_ratio = max_policy_perf; + } else { + int turbo_max = cpu->pstate.turbo_pstate; + int32_t global_min, global_max; + + /* Global limits are in percent of the maximum turbo P-state. */ + global_max = DIV_ROUND_UP(turbo_max * global.max_perf_pct, 100); + global_min = DIV_ROUND_UP(turbo_max * global.min_perf_pct, 100); + global_min = clamp_t(int32_t, global_min, 0, global_max); + + pr_debug("cpu:%d global_min:%d global_max:%d\n", cpu->cpu, + global_min, global_max); + + cpu->min_perf_ratio = max(min_policy_perf, global_min); + cpu->min_perf_ratio = min(cpu->min_perf_ratio, max_policy_perf); + cpu->max_perf_ratio = min(max_policy_perf, global_max); + cpu->max_perf_ratio = max(min_policy_perf, cpu->max_perf_ratio); + + /* Make sure min_perf <= max_perf */ + cpu->min_perf_ratio = min(cpu->min_perf_ratio, + cpu->max_perf_ratio); + + } + pr_debug("cpu:%d max_perf_ratio:%d min_perf_ratio:%d\n", cpu->cpu, + cpu->max_perf_ratio, + cpu->min_perf_ratio); } static int intel_pstate_set_policy(struct cpufreq_policy *policy) { struct cpudata *cpu; - cpu = all_cpu_data[policy->cpu]; - if (!policy->cpuinfo.max_freq) return -ENODEV; - if (policy->policy == CPUFREQ_POLICY_PERFORMANCE) { - limits.min_perf_pct = 100; - limits.min_perf = int_tofp(1); - limits.max_perf_pct = 100; - limits.max_perf = int_tofp(1); - limits.no_turbo = 0; - return 0; + pr_debug("set_policy cpuinfo.max %u policy->max %u\n", + policy->cpuinfo.max_freq, policy->max); + + cpu = all_cpu_data[policy->cpu]; + cpu->policy = policy->policy; + + mutex_lock(&intel_pstate_limits_lock); + + intel_pstate_update_perf_limits(cpu, policy->min, policy->max); + + if (cpu->policy == CPUFREQ_POLICY_PERFORMANCE) { + int pstate = max(cpu->pstate.min_pstate, cpu->max_perf_ratio); + + /* + * NOHZ_FULL CPUs need this as the governor callback may not + * be invoked on them. + */ + intel_pstate_clear_update_util_hook(policy->cpu); + intel_pstate_set_pstate(cpu, pstate); + } else { + intel_pstate_set_update_util_hook(policy->cpu); + } + + if (hwp_active) { + /* + * When hwp_boost was active before and dynamically it + * was turned off, in that case we need to clear the + * update util hook. + */ + if (!hwp_boost) + intel_pstate_clear_update_util_hook(policy->cpu); + intel_pstate_hwp_set(policy->cpu); } - limits.min_perf_pct = (policy->min * 100) / policy->cpuinfo.max_freq; - limits.min_perf_pct = clamp_t(int, limits.min_perf_pct, 0 , 100); - limits.min_perf = div_fp(int_tofp(limits.min_perf_pct), int_tofp(100)); + /* + * policy->cur is never updated with the intel_pstate driver, but it + * is used as a stale frequency value. So, keep it within limits. + */ + policy->cur = policy->min; - limits.max_policy_pct = policy->max * 100 / policy->cpuinfo.max_freq; - limits.max_policy_pct = clamp_t(int, limits.max_policy_pct, 0 , 100); - limits.max_perf_pct = min(limits.max_policy_pct, limits.max_sysfs_pct); - limits.max_perf = div_fp(int_tofp(limits.max_perf_pct), int_tofp(100)); + mutex_unlock(&intel_pstate_limits_lock); return 0; } -static int intel_pstate_verify_policy(struct cpufreq_policy *policy) +static void intel_pstate_adjust_policy_max(struct cpudata *cpu, + struct cpufreq_policy_data *policy) { - cpufreq_verify_within_limits(policy, - policy->cpuinfo.min_freq, - policy->cpuinfo.max_freq); + if (!hwp_active && + cpu->pstate.max_pstate_physical > cpu->pstate.max_pstate && + policy->max < policy->cpuinfo.max_freq && + policy->max > cpu->pstate.max_freq) { + pr_debug("policy->max > max non turbo frequency\n"); + policy->max = policy->cpuinfo.max_freq; + } +} - if ((policy->policy != CPUFREQ_POLICY_POWERSAVE) && - (policy->policy != CPUFREQ_POLICY_PERFORMANCE)) - return -EINVAL; +static void intel_pstate_verify_cpu_policy(struct cpudata *cpu, + struct cpufreq_policy_data *policy) +{ + int max_freq; + + if (hwp_active) { + intel_pstate_get_hwp_cap(cpu); + max_freq = READ_ONCE(global.no_turbo) ? + cpu->pstate.max_freq : cpu->pstate.turbo_freq; + } else { + max_freq = intel_pstate_get_max_freq(cpu); + } + cpufreq_verify_within_limits(policy, policy->cpuinfo.min_freq, max_freq); + + intel_pstate_adjust_policy_max(cpu, policy); +} + +static int intel_pstate_verify_policy(struct cpufreq_policy_data *policy) +{ + intel_pstate_verify_cpu_policy(all_cpu_data[policy->cpu], policy); return 0; } -static int __cpuinit intel_pstate_cpu_exit(struct cpufreq_policy *policy) +static int intel_cpufreq_cpu_offline(struct cpufreq_policy *policy) { - int cpu = policy->cpu; + struct cpudata *cpu = all_cpu_data[policy->cpu]; + + pr_debug("CPU %d going offline\n", cpu->cpu); + + if (cpu->suspended) + return 0; + + /* + * If the CPU is an SMT thread and it goes offline with the performance + * settings different from the minimum, it will prevent its sibling + * from getting to lower performance levels, so force the minimum + * performance on CPU offline to prevent that from happening. + */ + if (hwp_active) + intel_pstate_hwp_offline(cpu); + else + intel_pstate_set_min_pstate(cpu); + + intel_pstate_exit_perf_limits(policy); - del_timer(&all_cpu_data[cpu]->timer); - kfree(all_cpu_data[cpu]); - all_cpu_data[cpu] = NULL; return 0; } -static int __cpuinit intel_pstate_cpu_init(struct cpufreq_policy *policy) +static int intel_pstate_cpu_online(struct cpufreq_policy *policy) +{ + struct cpudata *cpu = all_cpu_data[policy->cpu]; + + pr_debug("CPU %d going online\n", cpu->cpu); + + intel_pstate_init_acpi_perf_limits(policy); + + if (hwp_active) { + /* + * Re-enable HWP and clear the "suspended" flag to let "resume" + * know that it need not do that. + */ + intel_pstate_hwp_reenable(cpu); + cpu->suspended = false; + + hybrid_update_capacity(cpu); + } + + return 0; +} + +static int intel_pstate_cpu_offline(struct cpufreq_policy *policy) +{ + intel_pstate_clear_update_util_hook(policy->cpu); + + return intel_cpufreq_cpu_offline(policy); +} + +static void intel_pstate_cpu_exit(struct cpufreq_policy *policy) +{ + pr_debug("CPU %d exiting\n", policy->cpu); + + policy->fast_switch_possible = false; +} + +static int __intel_pstate_cpu_init(struct cpufreq_policy *policy) { - int rc, min_pstate, max_pstate; struct cpudata *cpu; + int rc; rc = intel_pstate_init_cpu(policy->cpu); if (rc) @@ -638,101 +3039,871 @@ static int __cpuinit intel_pstate_cpu_init(struct cpufreq_policy *policy) cpu = all_cpu_data[policy->cpu]; - if (!limits.no_turbo && - limits.min_perf_pct == 100 && limits.max_perf_pct == 100) - policy->policy = CPUFREQ_POLICY_PERFORMANCE; - else - policy->policy = CPUFREQ_POLICY_POWERSAVE; - - intel_pstate_get_min_max(cpu, &min_pstate, &max_pstate); - policy->min = min_pstate * 100000; - policy->max = max_pstate * 100000; + cpu->max_perf_ratio = 0xFF; + cpu->min_perf_ratio = 0; /* cpuinfo and default policy values */ - policy->cpuinfo.min_freq = cpu->pstate.min_pstate * 100000; - policy->cpuinfo.max_freq = cpu->pstate.turbo_pstate * 100000; - policy->cpuinfo.transition_latency = CPUFREQ_ETERNAL; - cpumask_set_cpu(policy->cpu, policy->cpus); + policy->cpuinfo.min_freq = cpu->pstate.min_freq; + policy->cpuinfo.max_freq = READ_ONCE(global.no_turbo) ? + cpu->pstate.max_freq : cpu->pstate.turbo_freq; + + policy->min = policy->cpuinfo.min_freq; + policy->max = policy->cpuinfo.max_freq; + + intel_pstate_init_acpi_perf_limits(policy); + + policy->fast_switch_possible = true; + + return 0; +} + +static int intel_pstate_cpu_init(struct cpufreq_policy *policy) +{ + int ret = __intel_pstate_cpu_init(policy); + + if (ret) + return ret; + + /* + * Set the policy to powersave to provide a valid fallback value in case + * the default cpufreq governor is neither powersave nor performance. + */ + policy->policy = CPUFREQ_POLICY_POWERSAVE; + + if (hwp_active) { + struct cpudata *cpu = all_cpu_data[policy->cpu]; + + cpu->epp_cached = intel_pstate_get_epp(cpu, 0); + } return 0; } -static struct cpufreq_driver intel_pstate_driver = { +static struct cpufreq_driver intel_pstate = { .flags = CPUFREQ_CONST_LOOPS, .verify = intel_pstate_verify_policy, .setpolicy = intel_pstate_set_policy, - .get = intel_pstate_get, + .suspend = intel_pstate_suspend, + .resume = intel_pstate_resume, .init = intel_pstate_cpu_init, .exit = intel_pstate_cpu_exit, + .offline = intel_pstate_cpu_offline, + .online = intel_pstate_cpu_online, + .update_limits = intel_pstate_update_limits, .name = "intel_pstate", - .owner = THIS_MODULE, }; -static int __initdata no_load; +static int intel_cpufreq_verify_policy(struct cpufreq_policy_data *policy) +{ + struct cpudata *cpu = all_cpu_data[policy->cpu]; + + intel_pstate_verify_cpu_policy(cpu, policy); + intel_pstate_update_perf_limits(cpu, policy->min, policy->max); -static int intel_pstate_msrs_not_valid(void) + return 0; +} + +/* Use of trace in passive mode: + * + * In passive mode the trace core_busy field (also known as the + * performance field, and lablelled as such on the graphs; also known as + * core_avg_perf) is not needed and so is re-assigned to indicate if the + * driver call was via the normal or fast switch path. Various graphs + * output from the intel_pstate_tracer.py utility that include core_busy + * (or performance or core_avg_perf) have a fixed y-axis from 0 to 100%, + * so we use 10 to indicate the normal path through the driver, and + * 90 to indicate the fast switch path through the driver. + * The scaled_busy field is not used, and is set to 0. + */ + +#define INTEL_PSTATE_TRACE_TARGET 10 +#define INTEL_PSTATE_TRACE_FAST_SWITCH 90 + +static void intel_cpufreq_trace(struct cpudata *cpu, unsigned int trace_type, int old_pstate) { - /* Check that all the msr's we are using are valid. */ - u64 aperf, mperf, tmp; + struct sample *sample; - rdmsrl(MSR_IA32_APERF, aperf); - rdmsrl(MSR_IA32_MPERF, mperf); + if (!trace_pstate_sample_enabled()) + return; - if (!intel_pstate_min_pstate() || - !intel_pstate_max_pstate() || - !intel_pstate_turbo_pstate()) - return -ENODEV; + if (!intel_pstate_sample(cpu, ktime_get())) + return; + + sample = &cpu->sample; + trace_pstate_sample(trace_type, + 0, + old_pstate, + cpu->pstate.current_pstate, + sample->mperf, + sample->aperf, + sample->tsc, + get_avg_frequency(cpu), + fp_toint(cpu->iowait_boost * 100)); +} + +static void intel_cpufreq_hwp_update(struct cpudata *cpu, u32 min, u32 max, + u32 desired, bool fast_switch) +{ + u64 prev = READ_ONCE(cpu->hwp_req_cached), value = prev; + + value &= ~HWP_MIN_PERF(~0L); + value |= HWP_MIN_PERF(min); + + value &= ~HWP_MAX_PERF(~0L); + value |= HWP_MAX_PERF(max); + + value &= ~HWP_DESIRED_PERF(~0L); + value |= HWP_DESIRED_PERF(desired); + + if (value == prev) + return; + + WRITE_ONCE(cpu->hwp_req_cached, value); + if (fast_switch) + wrmsrq(MSR_HWP_REQUEST, value); + else + wrmsrq_on_cpu(cpu->cpu, MSR_HWP_REQUEST, value); +} + +static void intel_cpufreq_perf_ctl_update(struct cpudata *cpu, + u32 target_pstate, bool fast_switch) +{ + if (fast_switch) + wrmsrq(MSR_IA32_PERF_CTL, + pstate_funcs.get_val(cpu, target_pstate)); + else + wrmsrq_on_cpu(cpu->cpu, MSR_IA32_PERF_CTL, + pstate_funcs.get_val(cpu, target_pstate)); +} + +static int intel_cpufreq_update_pstate(struct cpufreq_policy *policy, + int target_pstate, bool fast_switch) +{ + struct cpudata *cpu = all_cpu_data[policy->cpu]; + int old_pstate = cpu->pstate.current_pstate; + + target_pstate = intel_pstate_prepare_request(cpu, target_pstate); + if (hwp_active) { + int max_pstate = policy->strict_target ? + target_pstate : cpu->max_perf_ratio; + + intel_cpufreq_hwp_update(cpu, target_pstate, max_pstate, + target_pstate, fast_switch); + } else if (target_pstate != old_pstate) { + intel_cpufreq_perf_ctl_update(cpu, target_pstate, fast_switch); + } + + cpu->pstate.current_pstate = target_pstate; + + intel_cpufreq_trace(cpu, fast_switch ? INTEL_PSTATE_TRACE_FAST_SWITCH : + INTEL_PSTATE_TRACE_TARGET, old_pstate); + + return target_pstate; +} + +static int intel_cpufreq_target(struct cpufreq_policy *policy, + unsigned int target_freq, + unsigned int relation) +{ + struct cpudata *cpu = all_cpu_data[policy->cpu]; + struct cpufreq_freqs freqs; + int target_pstate; + + freqs.old = policy->cur; + freqs.new = target_freq; + + cpufreq_freq_transition_begin(policy, &freqs); + + target_pstate = intel_pstate_freq_to_hwp_rel(cpu, freqs.new, relation); + target_pstate = intel_cpufreq_update_pstate(policy, target_pstate, false); + + freqs.new = target_pstate * cpu->pstate.scaling; + + cpufreq_freq_transition_end(policy, &freqs, false); + + return 0; +} + +static unsigned int intel_cpufreq_fast_switch(struct cpufreq_policy *policy, + unsigned int target_freq) +{ + struct cpudata *cpu = all_cpu_data[policy->cpu]; + int target_pstate; + + target_pstate = intel_pstate_freq_to_hwp(cpu, target_freq); + + target_pstate = intel_cpufreq_update_pstate(policy, target_pstate, true); + + return target_pstate * cpu->pstate.scaling; +} + +static void intel_cpufreq_adjust_perf(unsigned int cpunum, + unsigned long min_perf, + unsigned long target_perf, + unsigned long capacity) +{ + struct cpudata *cpu = all_cpu_data[cpunum]; + u64 hwp_cap = READ_ONCE(cpu->hwp_cap_cached); + int old_pstate = cpu->pstate.current_pstate; + int cap_pstate, min_pstate, max_pstate, target_pstate; + + cap_pstate = READ_ONCE(global.no_turbo) ? + HWP_GUARANTEED_PERF(hwp_cap) : + HWP_HIGHEST_PERF(hwp_cap); + + /* Optimization: Avoid unnecessary divisions. */ + + target_pstate = cap_pstate; + if (target_perf < capacity) + target_pstate = DIV_ROUND_UP(cap_pstate * target_perf, capacity); + + min_pstate = cap_pstate; + if (min_perf < capacity) + min_pstate = DIV_ROUND_UP(cap_pstate * min_perf, capacity); + + if (min_pstate < cpu->pstate.min_pstate) + min_pstate = cpu->pstate.min_pstate; + + if (min_pstate < cpu->min_perf_ratio) + min_pstate = cpu->min_perf_ratio; + + if (min_pstate > cpu->max_perf_ratio) + min_pstate = cpu->max_perf_ratio; - rdmsrl(MSR_IA32_APERF, tmp); - if (!(tmp - aperf)) + max_pstate = min(cap_pstate, cpu->max_perf_ratio); + if (max_pstate < min_pstate) + max_pstate = min_pstate; + + target_pstate = clamp_t(int, target_pstate, min_pstate, max_pstate); + + intel_cpufreq_hwp_update(cpu, min_pstate, max_pstate, target_pstate, true); + + cpu->pstate.current_pstate = target_pstate; + intel_cpufreq_trace(cpu, INTEL_PSTATE_TRACE_FAST_SWITCH, old_pstate); +} + +static int intel_cpufreq_cpu_init(struct cpufreq_policy *policy) +{ + struct freq_qos_request *req; + struct cpudata *cpu; + struct device *dev; + int ret, freq; + + dev = get_cpu_device(policy->cpu); + if (!dev) return -ENODEV; - rdmsrl(MSR_IA32_MPERF, tmp); - if (!(tmp - mperf)) + ret = __intel_pstate_cpu_init(policy); + if (ret) + return ret; + + policy->cpuinfo.transition_latency = INTEL_CPUFREQ_TRANSITION_LATENCY; + /* This reflects the intel_pstate_get_cpu_pstates() setting. */ + policy->cur = policy->cpuinfo.min_freq; + + req = kcalloc(2, sizeof(*req), GFP_KERNEL); + if (!req) { + ret = -ENOMEM; + goto pstate_exit; + } + + cpu = all_cpu_data[policy->cpu]; + + if (hwp_active) { + u64 value; + + policy->transition_delay_us = INTEL_CPUFREQ_TRANSITION_DELAY_HWP; + + intel_pstate_get_hwp_cap(cpu); + + rdmsrq_on_cpu(cpu->cpu, MSR_HWP_REQUEST, &value); + WRITE_ONCE(cpu->hwp_req_cached, value); + + cpu->epp_cached = intel_pstate_get_epp(cpu, value); + } else { + policy->transition_delay_us = INTEL_CPUFREQ_TRANSITION_DELAY; + } + + freq = DIV_ROUND_UP(cpu->pstate.turbo_freq * global.min_perf_pct, 100); + + ret = freq_qos_add_request(&policy->constraints, req, FREQ_QOS_MIN, + freq); + if (ret < 0) { + dev_err(dev, "Failed to add min-freq constraint (%d)\n", ret); + goto free_req; + } + + freq = DIV_ROUND_UP(cpu->pstate.turbo_freq * global.max_perf_pct, 100); + + ret = freq_qos_add_request(&policy->constraints, req + 1, FREQ_QOS_MAX, + freq); + if (ret < 0) { + dev_err(dev, "Failed to add max-freq constraint (%d)\n", ret); + goto remove_min_req; + } + + policy->driver_data = req; + + return 0; + +remove_min_req: + freq_qos_remove_request(req); +free_req: + kfree(req); +pstate_exit: + intel_pstate_exit_perf_limits(policy); + + return ret; +} + +static void intel_cpufreq_cpu_exit(struct cpufreq_policy *policy) +{ + struct freq_qos_request *req; + + req = policy->driver_data; + + freq_qos_remove_request(req + 1); + freq_qos_remove_request(req); + kfree(req); + + intel_pstate_cpu_exit(policy); +} + +static int intel_cpufreq_suspend(struct cpufreq_policy *policy) +{ + intel_pstate_suspend(policy); + + if (hwp_active) { + struct cpudata *cpu = all_cpu_data[policy->cpu]; + u64 value = READ_ONCE(cpu->hwp_req_cached); + + /* + * Clear the desired perf field in MSR_HWP_REQUEST in case + * intel_cpufreq_adjust_perf() is in use and the last value + * written by it may not be suitable. + */ + value &= ~HWP_DESIRED_PERF(~0L); + wrmsrq_on_cpu(cpu->cpu, MSR_HWP_REQUEST, value); + WRITE_ONCE(cpu->hwp_req_cached, value); + } + + return 0; +} + +static struct cpufreq_driver intel_cpufreq = { + .flags = CPUFREQ_CONST_LOOPS, + .verify = intel_cpufreq_verify_policy, + .target = intel_cpufreq_target, + .fast_switch = intel_cpufreq_fast_switch, + .init = intel_cpufreq_cpu_init, + .exit = intel_cpufreq_cpu_exit, + .offline = intel_cpufreq_cpu_offline, + .online = intel_pstate_cpu_online, + .suspend = intel_cpufreq_suspend, + .resume = intel_pstate_resume, + .update_limits = intel_pstate_update_limits, + .name = "intel_cpufreq", +}; + +static struct cpufreq_driver *default_driver; + +static void intel_pstate_driver_cleanup(void) +{ + unsigned int cpu; + + cpus_read_lock(); + for_each_online_cpu(cpu) { + if (all_cpu_data[cpu]) { + if (intel_pstate_driver == &intel_pstate) + intel_pstate_clear_update_util_hook(cpu); + + kfree(all_cpu_data[cpu]); + WRITE_ONCE(all_cpu_data[cpu], NULL); + } + } + cpus_read_unlock(); + + intel_pstate_driver = NULL; +} + +static int intel_pstate_register_driver(struct cpufreq_driver *driver) +{ + bool refresh_cpu_cap_scaling; + int ret; + + if (driver == &intel_pstate) + intel_pstate_sysfs_expose_hwp_dynamic_boost(); + + memset(&global, 0, sizeof(global)); + global.max_perf_pct = 100; + global.turbo_disabled = turbo_is_disabled(); + global.no_turbo = global.turbo_disabled; + + arch_set_max_freq_ratio(global.turbo_disabled); + + refresh_cpu_cap_scaling = hybrid_clear_max_perf_cpu(); + + intel_pstate_driver = driver; + ret = cpufreq_register_driver(intel_pstate_driver); + if (ret) { + intel_pstate_driver_cleanup(); + return ret; + } + + global.min_perf_pct = min_perf_pct_min(); + + hybrid_init_cpu_capacity_scaling(refresh_cpu_cap_scaling); + + return 0; +} + +static ssize_t intel_pstate_show_status(char *buf) +{ + if (!intel_pstate_driver) + return sprintf(buf, "off\n"); + + return sprintf(buf, "%s\n", intel_pstate_driver == &intel_pstate ? + "active" : "passive"); +} + +static int intel_pstate_update_status(const char *buf, size_t size) +{ + if (size == 3 && !strncmp(buf, "off", size)) { + if (!intel_pstate_driver) + return -EINVAL; + + if (hwp_active) + return -EBUSY; + + cpufreq_unregister_driver(intel_pstate_driver); + intel_pstate_driver_cleanup(); + return 0; + } + + if (size == 6 && !strncmp(buf, "active", size)) { + if (intel_pstate_driver) { + if (intel_pstate_driver == &intel_pstate) + return 0; + + cpufreq_unregister_driver(intel_pstate_driver); + } + + return intel_pstate_register_driver(&intel_pstate); + } + + if (size == 7 && !strncmp(buf, "passive", size)) { + if (intel_pstate_driver) { + if (intel_pstate_driver == &intel_cpufreq) + return 0; + + cpufreq_unregister_driver(intel_pstate_driver); + intel_pstate_sysfs_hide_hwp_dynamic_boost(); + } + + return intel_pstate_register_driver(&intel_cpufreq); + } + + return -EINVAL; +} + +static int no_load __initdata; +static int no_hwp __initdata; +static int hwp_only __initdata; +static unsigned int force_load __initdata; + +static int __init intel_pstate_msrs_not_valid(void) +{ + if (!pstate_funcs.get_max(0) || + !pstate_funcs.get_min(0) || + !pstate_funcs.get_turbo(0)) return -ENODEV; return 0; } + +static void __init copy_cpu_funcs(struct pstate_funcs *funcs) +{ + pstate_funcs.get_max = funcs->get_max; + pstate_funcs.get_max_physical = funcs->get_max_physical; + pstate_funcs.get_min = funcs->get_min; + pstate_funcs.get_turbo = funcs->get_turbo; + pstate_funcs.get_scaling = funcs->get_scaling; + pstate_funcs.get_val = funcs->get_val; + pstate_funcs.get_vid = funcs->get_vid; + pstate_funcs.get_aperf_mperf_shift = funcs->get_aperf_mperf_shift; +} + +#ifdef CONFIG_ACPI + +static bool __init intel_pstate_no_acpi_pss(void) +{ + int i; + + for_each_possible_cpu(i) { + acpi_status status; + union acpi_object *pss; + struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL }; + struct acpi_processor *pr = per_cpu(processors, i); + + if (!pr) + continue; + + status = acpi_evaluate_object(pr->handle, "_PSS", NULL, &buffer); + if (ACPI_FAILURE(status)) + continue; + + pss = buffer.pointer; + if (pss && pss->type == ACPI_TYPE_PACKAGE) { + kfree(pss); + return false; + } + + kfree(pss); + } + + pr_debug("ACPI _PSS not found\n"); + return true; +} + +static bool __init intel_pstate_no_acpi_pcch(void) +{ + acpi_status status; + acpi_handle handle; + + status = acpi_get_handle(NULL, "\\_SB", &handle); + if (ACPI_FAILURE(status)) + goto not_found; + + if (acpi_has_method(handle, "PCCH")) + return false; + +not_found: + pr_debug("ACPI PCCH not found\n"); + return true; +} + +static bool __init intel_pstate_has_acpi_ppc(void) +{ + int i; + + for_each_possible_cpu(i) { + struct acpi_processor *pr = per_cpu(processors, i); + + if (!pr) + continue; + if (acpi_has_method(pr->handle, "_PPC")) + return true; + } + pr_debug("ACPI _PPC not found\n"); + return false; +} + +enum { + PSS, + PPC, +}; + +/* Hardware vendor-specific info that has its own power management modes */ +static struct acpi_platform_list plat_info[] __initdata = { + {"HP ", "ProLiant", 0, ACPI_SIG_FADT, all_versions, NULL, PSS}, + {"ORACLE", "X4-2 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC}, + {"ORACLE", "X4-2L ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC}, + {"ORACLE", "X4-2B ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC}, + {"ORACLE", "X3-2 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC}, + {"ORACLE", "X3-2L ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC}, + {"ORACLE", "X3-2B ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC}, + {"ORACLE", "X4470M2 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC}, + {"ORACLE", "X4270M3 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC}, + {"ORACLE", "X4270M2 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC}, + {"ORACLE", "X4170M2 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC}, + {"ORACLE", "X4170 M3", 0, ACPI_SIG_FADT, all_versions, NULL, PPC}, + {"ORACLE", "X4275 M3", 0, ACPI_SIG_FADT, all_versions, NULL, PPC}, + {"ORACLE", "X6-2 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC}, + {"ORACLE", "Sudbury ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC}, + { } /* End */ +}; + +#define BITMASK_OOB (BIT(8) | BIT(18)) + +static bool __init intel_pstate_platform_pwr_mgmt_exists(void) +{ + const struct x86_cpu_id *id; + u64 misc_pwr; + int idx; + + id = x86_match_cpu(intel_pstate_cpu_oob_ids); + if (id) { + rdmsrq(MSR_MISC_PWR_MGMT, misc_pwr); + if (misc_pwr & BITMASK_OOB) { + pr_debug("Bit 8 or 18 in the MISC_PWR_MGMT MSR set\n"); + pr_debug("P states are controlled in Out of Band mode by the firmware/hardware\n"); + return true; + } + } + + idx = acpi_match_platform_list(plat_info); + if (idx < 0) + return false; + + switch (plat_info[idx].data) { + case PSS: + if (!intel_pstate_no_acpi_pss()) + return false; + + return intel_pstate_no_acpi_pcch(); + case PPC: + return intel_pstate_has_acpi_ppc() && !force_load; + } + + return false; +} + +static void intel_pstate_request_control_from_smm(void) +{ + /* + * It may be unsafe to request P-states control from SMM if _PPC support + * has not been enabled. + */ + if (acpi_ppc) + acpi_processor_pstate_control(); +} +#else /* CONFIG_ACPI not enabled */ +static inline bool intel_pstate_platform_pwr_mgmt_exists(void) { return false; } +static inline bool intel_pstate_has_acpi_ppc(void) { return false; } +static inline void intel_pstate_request_control_from_smm(void) {} +#endif /* CONFIG_ACPI */ + +#define INTEL_PSTATE_HWP_BROADWELL 0x01 + +#define X86_MATCH_HWP(vfm, hwp_mode) \ + X86_MATCH_VFM_FEATURE(vfm, X86_FEATURE_HWP, hwp_mode) + +static const struct x86_cpu_id hwp_support_ids[] __initconst = { + X86_MATCH_HWP(INTEL_BROADWELL_X, INTEL_PSTATE_HWP_BROADWELL), + X86_MATCH_HWP(INTEL_BROADWELL_D, INTEL_PSTATE_HWP_BROADWELL), + X86_MATCH_HWP(INTEL_ANY, 0), + {} +}; + +static bool intel_pstate_hwp_is_enabled(void) +{ + u64 value; + + rdmsrq(MSR_PM_ENABLE, value); + return !!(value & 0x1); +} + +#define POWERSAVE_MASK GENMASK(7, 0) +#define BALANCE_POWER_MASK GENMASK(15, 8) +#define BALANCE_PERFORMANCE_MASK GENMASK(23, 16) +#define PERFORMANCE_MASK GENMASK(31, 24) + +#define HWP_SET_EPP_VALUES(powersave, balance_power, balance_perf, performance) \ + (FIELD_PREP_CONST(POWERSAVE_MASK, powersave) |\ + FIELD_PREP_CONST(BALANCE_POWER_MASK, balance_power) |\ + FIELD_PREP_CONST(BALANCE_PERFORMANCE_MASK, balance_perf) |\ + FIELD_PREP_CONST(PERFORMANCE_MASK, performance)) + +#define HWP_SET_DEF_BALANCE_PERF_EPP(balance_perf) \ + (HWP_SET_EPP_VALUES(HWP_EPP_POWERSAVE, HWP_EPP_BALANCE_POWERSAVE,\ + balance_perf, HWP_EPP_PERFORMANCE)) + +static const struct x86_cpu_id intel_epp_default[] = { + /* + * Set EPP value as 102, this is the max suggested EPP + * which can result in one core turbo frequency for + * AlderLake Mobile CPUs. + */ + X86_MATCH_VFM(INTEL_ALDERLAKE_L, HWP_SET_DEF_BALANCE_PERF_EPP(102)), + X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X, HWP_SET_DEF_BALANCE_PERF_EPP(32)), + X86_MATCH_VFM(INTEL_EMERALDRAPIDS_X, HWP_SET_DEF_BALANCE_PERF_EPP(32)), + X86_MATCH_VFM(INTEL_GRANITERAPIDS_X, HWP_SET_DEF_BALANCE_PERF_EPP(32)), + X86_MATCH_VFM(INTEL_GRANITERAPIDS_D, HWP_SET_DEF_BALANCE_PERF_EPP(32)), + X86_MATCH_VFM(INTEL_METEORLAKE_L, HWP_SET_EPP_VALUES(HWP_EPP_POWERSAVE, + 179, 64, 16)), + X86_MATCH_VFM(INTEL_ARROWLAKE, HWP_SET_EPP_VALUES(HWP_EPP_POWERSAVE, + 179, 64, 16)), + {} +}; + +static const struct x86_cpu_id intel_hybrid_scaling_factor[] = { + X86_MATCH_VFM(INTEL_ALDERLAKE, HYBRID_SCALING_FACTOR_ADL), + X86_MATCH_VFM(INTEL_ALDERLAKE_L, HYBRID_SCALING_FACTOR_ADL), + X86_MATCH_VFM(INTEL_RAPTORLAKE, HYBRID_SCALING_FACTOR_ADL), + X86_MATCH_VFM(INTEL_RAPTORLAKE_P, HYBRID_SCALING_FACTOR_ADL), + X86_MATCH_VFM(INTEL_RAPTORLAKE_S, HYBRID_SCALING_FACTOR_ADL), + X86_MATCH_VFM(INTEL_METEORLAKE_L, HYBRID_SCALING_FACTOR_MTL), + X86_MATCH_VFM(INTEL_LUNARLAKE_M, HYBRID_SCALING_FACTOR_LNL), + {} +}; + +static bool hwp_check_epp(void) +{ + if (boot_cpu_has(X86_FEATURE_HWP_EPP)) + return true; + + /* Without EPP support, don't expose EPP-related sysfs attributes. */ + hwp_cpufreq_attrs[HWP_PERFORMANCE_PREFERENCE_INDEX] = NULL; + hwp_cpufreq_attrs[HWP_PERFORMANCE_AVAILABLE_PREFERENCES_INDEX] = NULL; + + return false; +} + +static bool hwp_check_dec(void) +{ + u64 power_ctl; + + rdmsrq(MSR_IA32_POWER_CTL, power_ctl); + return !!(power_ctl & BIT(POWER_CTL_DEC_ENABLE)); +} + static int __init intel_pstate_init(void) { - int cpu, rc = 0; + static struct cpudata **_all_cpu_data; const struct x86_cpu_id *id; + int rc; - if (no_load) + if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) return -ENODEV; - id = x86_match_cpu(intel_pstate_cpu_ids); - if (!id) + /* + * The Intel pstate driver will be ignored if the platform + * firmware has its own power management modes. + */ + if (intel_pstate_platform_pwr_mgmt_exists()) { + pr_info("P-states controlled by the platform\n"); return -ENODEV; + } + + id = x86_match_cpu(hwp_support_ids); + if (id) { + bool epp_present = hwp_check_epp(); + + /* + * If HWP is enabled already, there is no choice but to deal + * with it. + */ + hwp_forced = intel_pstate_hwp_is_enabled(); + if (hwp_forced) { + pr_info("HWP enabled by BIOS\n"); + no_hwp = 0; + } else if (no_load) { + return -ENODEV; + } else if (!epp_present && !hwp_check_dec()) { + /* + * Avoid enabling HWP for processors without EPP support + * unless the Dynamic Efficiency Control (DEC) enable + * bit (MSR_IA32_POWER_CTL, bit 27) is set because that + * means incomplete HWP implementation which is a corner + * case and supporting it is generally problematic. + */ + no_hwp = 1; + } + + copy_cpu_funcs(&core_funcs); + + if (!no_hwp) { + hwp_active = true; + hwp_mode_bdw = id->driver_data; + intel_pstate.attr = hwp_cpufreq_attrs; + intel_cpufreq.attr = hwp_cpufreq_attrs; + intel_cpufreq.flags |= CPUFREQ_NEED_UPDATE_LIMITS; + intel_cpufreq.adjust_perf = intel_cpufreq_adjust_perf; + if (!default_driver) + default_driver = &intel_pstate; + + pstate_funcs.get_cpu_scaling = hwp_get_cpu_scaling; + + goto hwp_cpu_matched; + } + pr_info("HWP not enabled\n"); + } else { + if (no_load) + return -ENODEV; + + id = x86_match_cpu(intel_pstate_cpu_ids); + if (!id) { + pr_info("CPU model not supported\n"); + return -ENODEV; + } - if (intel_pstate_msrs_not_valid()) + copy_cpu_funcs((struct pstate_funcs *)id->driver_data); + } + + if (intel_pstate_msrs_not_valid()) { + pr_info("Invalid MSRs\n"); return -ENODEV; + } + /* Without HWP start in the passive mode. */ + if (!default_driver) + default_driver = &intel_cpufreq; - pr_info("Intel P-state driver initializing.\n"); +hwp_cpu_matched: + if (!hwp_active && hwp_only) + return -ENOTSUPP; - all_cpu_data = vzalloc(sizeof(void *) * num_possible_cpus()); - if (!all_cpu_data) + pr_info("Intel P-state driver initializing\n"); + + _all_cpu_data = vzalloc(array_size(sizeof(void *), num_possible_cpus())); + if (!_all_cpu_data) return -ENOMEM; - rc = cpufreq_register_driver(&intel_pstate_driver); - if (rc) - goto out; + WRITE_ONCE(all_cpu_data, _all_cpu_data); + + intel_pstate_request_control_from_smm(); - intel_pstate_debug_expose_params(); intel_pstate_sysfs_expose_params(); - return rc; -out: - get_online_cpus(); - for_each_online_cpu(cpu) { - if (all_cpu_data[cpu]) { - del_timer_sync(&all_cpu_data[cpu]->timer); - kfree(all_cpu_data[cpu]); + + if (hwp_active) { + const struct x86_cpu_id *id = x86_match_cpu(intel_epp_default); + const struct x86_cpu_id *hybrid_id = x86_match_cpu(intel_hybrid_scaling_factor); + + if (id) { + epp_values[EPP_INDEX_POWERSAVE] = + FIELD_GET(POWERSAVE_MASK, id->driver_data); + epp_values[EPP_INDEX_BALANCE_POWERSAVE] = + FIELD_GET(BALANCE_POWER_MASK, id->driver_data); + epp_values[EPP_INDEX_BALANCE_PERFORMANCE] = + FIELD_GET(BALANCE_PERFORMANCE_MASK, id->driver_data); + epp_values[EPP_INDEX_PERFORMANCE] = + FIELD_GET(PERFORMANCE_MASK, id->driver_data); + pr_debug("Updated EPPs powersave:%x balanced power:%x balanced perf:%x performance:%x\n", + epp_values[EPP_INDEX_POWERSAVE], + epp_values[EPP_INDEX_BALANCE_POWERSAVE], + epp_values[EPP_INDEX_BALANCE_PERFORMANCE], + epp_values[EPP_INDEX_PERFORMANCE]); + } + + if (hybrid_id) { + hybrid_scaling_factor = hybrid_id->driver_data; + pr_debug("hybrid scaling factor: %d\n", hybrid_scaling_factor); } + + } + + scoped_guard(mutex, &intel_pstate_driver_lock) { + rc = intel_pstate_register_driver(default_driver); + } + if (rc) { + intel_pstate_sysfs_remove(); + return rc; } - put_online_cpus(); - vfree(all_cpu_data); - return -ENODEV; + if (hwp_active) { + const struct x86_cpu_id *id; + + id = x86_match_cpu(intel_pstate_cpu_ee_disable_ids); + if (id) { + set_power_ctl_ee_state(false); + pr_info("Disabling energy efficiency optimization\n"); + } + + pr_info("HWP enabled\n"); + } else if (boot_cpu_has(X86_FEATURE_HYBRID_CPU)) { + pr_warn("Problematic setup: Hybrid processor with disabled HWP\n"); + } + + return 0; } device_initcall(intel_pstate_init); @@ -743,10 +3914,32 @@ static int __init intel_pstate_setup(char *str) if (!strcmp(str, "disable")) no_load = 1; + else if (!strcmp(str, "active")) + default_driver = &intel_pstate; + else if (!strcmp(str, "passive")) + default_driver = &intel_cpufreq; + + if (!strcmp(str, "no_hwp")) + no_hwp = 1; + + if (!strcmp(str, "no_cas")) + no_cas = true; + + if (!strcmp(str, "force")) + force_load = 1; + if (!strcmp(str, "hwp_only")) + hwp_only = 1; + if (!strcmp(str, "per_cpu_perf_limits")) + per_cpu_limits = true; + +#ifdef CONFIG_ACPI + if (!strcmp(str, "support_acpi_ppc")) + acpi_ppc = true; +#endif + return 0; } early_param("intel_pstate", intel_pstate_setup); MODULE_AUTHOR("Dirk Brandewie <dirk.j.brandewie@intel.com>"); MODULE_DESCRIPTION("'intel_pstate' - P state driver Intel Core processors"); -MODULE_LICENSE("GPL"); diff --git a/drivers/cpufreq/kirkwood-cpufreq.c b/drivers/cpufreq/kirkwood-cpufreq.c index c233ea617366..24b285cbeb8d 100644 --- a/drivers/cpufreq/kirkwood-cpufreq.c +++ b/drivers/cpufreq/kirkwood-cpufreq.c @@ -1,18 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0-or-later /* * kirkwood_freq.c: cpufreq driver for the Marvell kirkwood * * Copyright (C) 2013 Andrew Lunn <andrew@lunn.ch> - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ #include <linux/kernel.h> #include <linux/module.h> #include <linux/clk.h> -#include <linux/clk-provider.h> #include <linux/cpufreq.h> #include <linux/of.h> #include <linux/platform_device.h> @@ -39,85 +34,50 @@ static struct priv * - cpu clk * - ddr clk * - * The frequencies are set at runtime before registering this * - * table. + * The frequencies are set at runtime before registering this table. */ static struct cpufreq_frequency_table kirkwood_freq_table[] = { - {STATE_CPU_FREQ, 0}, /* CPU uses cpuclk */ - {STATE_DDR_FREQ, 0}, /* CPU uses ddrclk */ - {0, CPUFREQ_TABLE_END}, + {0, STATE_CPU_FREQ, 0}, /* CPU uses cpuclk */ + {0, STATE_DDR_FREQ, 0}, /* CPU uses ddrclk */ + {0, 0, CPUFREQ_TABLE_END}, }; static unsigned int kirkwood_cpufreq_get_cpu_frequency(unsigned int cpu) { - if (__clk_is_enabled(priv.powersave_clk)) - return kirkwood_freq_table[1].frequency; - return kirkwood_freq_table[0].frequency; + return clk_get_rate(priv.powersave_clk) / 1000; } -static void kirkwood_cpufreq_set_cpu_state(struct cpufreq_policy *policy, - unsigned int index) +static int kirkwood_cpufreq_target(struct cpufreq_policy *policy, + unsigned int index) { - struct cpufreq_freqs freqs; unsigned int state = kirkwood_freq_table[index].driver_data; unsigned long reg; - freqs.old = kirkwood_cpufreq_get_cpu_frequency(0); - freqs.new = kirkwood_freq_table[index].frequency; - - cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE); - - dev_dbg(priv.dev, "Attempting to set frequency to %i KHz\n", - kirkwood_freq_table[index].frequency); - dev_dbg(priv.dev, "old frequency was %i KHz\n", - kirkwood_cpufreq_get_cpu_frequency(0)); - - if (freqs.old != freqs.new) { - local_irq_disable(); + local_irq_disable(); - /* Disable interrupts to the CPU */ - reg = readl_relaxed(priv.base); - reg |= CPU_SW_INT_BLK; - writel_relaxed(reg, priv.base); + /* Disable interrupts to the CPU */ + reg = readl_relaxed(priv.base); + reg |= CPU_SW_INT_BLK; + writel_relaxed(reg, priv.base); - switch (state) { - case STATE_CPU_FREQ: - clk_disable(priv.powersave_clk); - break; - case STATE_DDR_FREQ: - clk_enable(priv.powersave_clk); - break; - } - - /* Wait-for-Interrupt, while the hardware changes frequency */ - cpu_do_idle(); - - /* Enable interrupts to the CPU */ - reg = readl_relaxed(priv.base); - reg &= ~CPU_SW_INT_BLK; - writel_relaxed(reg, priv.base); - - local_irq_enable(); + switch (state) { + case STATE_CPU_FREQ: + clk_set_parent(priv.powersave_clk, priv.cpu_clk); + break; + case STATE_DDR_FREQ: + clk_set_parent(priv.powersave_clk, priv.ddr_clk); + break; } - cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); -}; -static int kirkwood_cpufreq_verify(struct cpufreq_policy *policy) -{ - return cpufreq_frequency_table_verify(policy, kirkwood_freq_table); -} + /* Wait-for-Interrupt, while the hardware changes frequency */ + cpu_do_idle(); -static int kirkwood_cpufreq_target(struct cpufreq_policy *policy, - unsigned int target_freq, - unsigned int relation) -{ - unsigned int index = 0; + /* Enable interrupts to the CPU */ + reg = readl_relaxed(priv.base); + reg &= ~CPU_SW_INT_BLK; + writel_relaxed(reg, priv.base); - if (cpufreq_frequency_table_target(policy, kirkwood_freq_table, - target_freq, relation, &index)) - return -EINVAL; - - kirkwood_cpufreq_set_cpu_state(policy, index); + local_irq_enable(); return 0; } @@ -125,115 +85,105 @@ static int kirkwood_cpufreq_target(struct cpufreq_policy *policy, /* Module init and exit code */ static int kirkwood_cpufreq_cpu_init(struct cpufreq_policy *policy) { - int result; - - /* cpuinfo and default policy values */ - policy->cpuinfo.transition_latency = 5000; /* 5uS */ - policy->cur = kirkwood_cpufreq_get_cpu_frequency(0); - - result = cpufreq_frequency_table_cpuinfo(policy, kirkwood_freq_table); - if (result) - return result; - - cpufreq_frequency_table_get_attr(kirkwood_freq_table, policy->cpu); - - return 0; -} - -static int kirkwood_cpufreq_cpu_exit(struct cpufreq_policy *policy) -{ - cpufreq_frequency_table_put_attr(policy->cpu); + cpufreq_generic_init(policy, kirkwood_freq_table, 5000); return 0; } -static struct freq_attr *kirkwood_cpufreq_attr[] = { - &cpufreq_freq_attr_scaling_available_freqs, - NULL, -}; - static struct cpufreq_driver kirkwood_cpufreq_driver = { + .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK, .get = kirkwood_cpufreq_get_cpu_frequency, - .verify = kirkwood_cpufreq_verify, - .target = kirkwood_cpufreq_target, + .verify = cpufreq_generic_frequency_table_verify, + .target_index = kirkwood_cpufreq_target, .init = kirkwood_cpufreq_cpu_init, - .exit = kirkwood_cpufreq_cpu_exit, .name = "kirkwood-cpufreq", - .owner = THIS_MODULE, - .attr = kirkwood_cpufreq_attr, }; static int kirkwood_cpufreq_probe(struct platform_device *pdev) { struct device_node *np; - struct resource *res; int err; priv.dev = &pdev->dev; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - priv.base = devm_ioremap_resource(&pdev->dev, res); + priv.base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(priv.base)) return PTR_ERR(priv.base); - np = of_find_node_by_path("/cpus/cpu@0"); - if (!np) + np = of_cpu_device_node_get(0); + if (!np) { + dev_err(&pdev->dev, "failed to get cpu device node\n"); return -ENODEV; + } priv.cpu_clk = of_clk_get_by_name(np, "cpu_clk"); if (IS_ERR(priv.cpu_clk)) { - dev_err(priv.dev, "Unable to get cpuclk"); - return PTR_ERR(priv.cpu_clk); + dev_err(priv.dev, "Unable to get cpuclk\n"); + err = PTR_ERR(priv.cpu_clk); + goto out_node; + } + + err = clk_prepare_enable(priv.cpu_clk); + if (err) { + dev_err(priv.dev, "Unable to prepare cpuclk\n"); + goto out_node; } - clk_prepare_enable(priv.cpu_clk); kirkwood_freq_table[0].frequency = clk_get_rate(priv.cpu_clk) / 1000; priv.ddr_clk = of_clk_get_by_name(np, "ddrclk"); if (IS_ERR(priv.ddr_clk)) { - dev_err(priv.dev, "Unable to get ddrclk"); + dev_err(priv.dev, "Unable to get ddrclk\n"); err = PTR_ERR(priv.ddr_clk); goto out_cpu; } - clk_prepare_enable(priv.ddr_clk); + err = clk_prepare_enable(priv.ddr_clk); + if (err) { + dev_err(priv.dev, "Unable to prepare ddrclk\n"); + goto out_cpu; + } kirkwood_freq_table[1].frequency = clk_get_rate(priv.ddr_clk) / 1000; priv.powersave_clk = of_clk_get_by_name(np, "powersave"); if (IS_ERR(priv.powersave_clk)) { - dev_err(priv.dev, "Unable to get powersave"); + dev_err(priv.dev, "Unable to get powersave\n"); err = PTR_ERR(priv.powersave_clk); goto out_ddr; } - clk_prepare(priv.powersave_clk); - - of_node_put(np); - np = NULL; + err = clk_prepare_enable(priv.powersave_clk); + if (err) { + dev_err(priv.dev, "Unable to prepare powersave clk\n"); + goto out_ddr; + } err = cpufreq_register_driver(&kirkwood_cpufreq_driver); - if (!err) - return 0; + if (err) { + dev_err(priv.dev, "Failed to register cpufreq driver\n"); + goto out_powersave; + } - dev_err(priv.dev, "Failed to register cpufreq driver"); + of_node_put(np); + return 0; +out_powersave: clk_disable_unprepare(priv.powersave_clk); out_ddr: clk_disable_unprepare(priv.ddr_clk); out_cpu: clk_disable_unprepare(priv.cpu_clk); +out_node: of_node_put(np); return err; } -static int kirkwood_cpufreq_remove(struct platform_device *pdev) +static void kirkwood_cpufreq_remove(struct platform_device *pdev) { cpufreq_unregister_driver(&kirkwood_cpufreq_driver); clk_disable_unprepare(priv.powersave_clk); clk_disable_unprepare(priv.ddr_clk); clk_disable_unprepare(priv.cpu_clk); - - return 0; } static struct platform_driver kirkwood_cpufreq_platform_driver = { @@ -241,7 +191,6 @@ static struct platform_driver kirkwood_cpufreq_platform_driver = { .remove = kirkwood_cpufreq_remove, .driver = { .name = "kirkwood-cpufreq", - .owner = THIS_MODULE, }, }; diff --git a/drivers/cpufreq/longhaul.c b/drivers/cpufreq/longhaul.c index b6a0a7a406b0..49e76b44468a 100644 --- a/drivers/cpufreq/longhaul.c +++ b/drivers/cpufreq/longhaul.c @@ -1,8 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0-only /* - * (C) 2001-2004 Dave Jones. <davej@redhat.com> + * (C) 2001-2004 Dave Jones. * (C) 2002 Padraig Brady. <padraig@antefacto.com> * - * Licensed under the terms of the GNU GPL License version 2. * Based upon datasheets & sample CPUs kindly provided by VIA. * * VIA have currently 3 different versions of Longhaul. @@ -21,6 +21,8 @@ * BIG FAT DISCLAIMER: Work in progress code. Possibly *dangerous* */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + #include <linux/kernel.h> #include <linux/module.h> #include <linux/moduleparam.h> @@ -40,8 +42,6 @@ #include "longhaul.h" -#define PFX "longhaul: " - #define TYPE_LONGHAUL_V1 1 #define TYPE_LONGHAUL_V2 2 #define TYPE_POWERSAVER 3 @@ -136,7 +136,7 @@ static void do_longhaul1(unsigned int mults_index) { union msr_bcr2 bcr2; - rdmsrl(MSR_VIA_BCR2, bcr2.val); + rdmsrq(MSR_VIA_BCR2, bcr2.val); /* Enable software clock multiplier */ bcr2.bits.ESOFTBF = 1; bcr2.bits.CLOCKMUL = mults_index & 0xff; @@ -144,16 +144,16 @@ static void do_longhaul1(unsigned int mults_index) /* Sync to timer tick */ safe_halt(); /* Change frequency on next halt or sleep */ - wrmsrl(MSR_VIA_BCR2, bcr2.val); + wrmsrq(MSR_VIA_BCR2, bcr2.val); /* Invoke transition */ ACPI_FLUSH_CPU_CACHE(); halt(); /* Disable software clock multiplier */ local_irq_disable(); - rdmsrl(MSR_VIA_BCR2, bcr2.val); + rdmsrq(MSR_VIA_BCR2, bcr2.val); bcr2.bits.ESOFTBF = 0; - wrmsrl(MSR_VIA_BCR2, bcr2.val); + wrmsrq(MSR_VIA_BCR2, bcr2.val); } /* For processor with Longhaul MSR */ @@ -164,7 +164,7 @@ static void do_powersaver(int cx_address, unsigned int mults_index, union msr_longhaul longhaul; u32 t; - rdmsrl(MSR_VIA_LONGHAUL, longhaul.val); + rdmsrq(MSR_VIA_LONGHAUL, longhaul.val); /* Setup new frequency */ if (!revid_errata) longhaul.bits.RevisionKey = longhaul.bits.RevisionID; @@ -180,7 +180,7 @@ static void do_powersaver(int cx_address, unsigned int mults_index, /* Raise voltage if necessary */ if (can_scale_voltage && dir) { longhaul.bits.EnableSoftVID = 1; - wrmsrl(MSR_VIA_LONGHAUL, longhaul.val); + wrmsrq(MSR_VIA_LONGHAUL, longhaul.val); /* Change voltage */ if (!cx_address) { ACPI_FLUSH_CPU_CACHE(); @@ -194,12 +194,12 @@ static void do_powersaver(int cx_address, unsigned int mults_index, t = inl(acpi_gbl_FADT.xpm_timer_block.address); } longhaul.bits.EnableSoftVID = 0; - wrmsrl(MSR_VIA_LONGHAUL, longhaul.val); + wrmsrq(MSR_VIA_LONGHAUL, longhaul.val); } /* Change frequency on next halt or sleep */ longhaul.bits.EnableSoftBusRatio = 1; - wrmsrl(MSR_VIA_LONGHAUL, longhaul.val); + wrmsrq(MSR_VIA_LONGHAUL, longhaul.val); if (!cx_address) { ACPI_FLUSH_CPU_CACHE(); halt(); @@ -212,12 +212,12 @@ static void do_powersaver(int cx_address, unsigned int mults_index, } /* Disable bus ratio bit */ longhaul.bits.EnableSoftBusRatio = 0; - wrmsrl(MSR_VIA_LONGHAUL, longhaul.val); + wrmsrq(MSR_VIA_LONGHAUL, longhaul.val); /* Reduce voltage if necessary */ if (can_scale_voltage && !dir) { longhaul.bits.EnableSoftVID = 1; - wrmsrl(MSR_VIA_LONGHAUL, longhaul.val); + wrmsrq(MSR_VIA_LONGHAUL, longhaul.val); /* Change voltage */ if (!cx_address) { ACPI_FLUSH_CPU_CACHE(); @@ -231,18 +231,19 @@ static void do_powersaver(int cx_address, unsigned int mults_index, t = inl(acpi_gbl_FADT.xpm_timer_block.address); } longhaul.bits.EnableSoftVID = 0; - wrmsrl(MSR_VIA_LONGHAUL, longhaul.val); + wrmsrq(MSR_VIA_LONGHAUL, longhaul.val); } } /** - * longhaul_set_cpu_frequency() - * @mults_index : bitpattern of the new multiplier. + * longhaul_setstate() + * @policy: cpufreq_policy structure containing the current policy. + * @table_index: index of the frequency within the cpufreq_frequency_table. * * Sets a new clock ratio. */ -static void longhaul_setstate(struct cpufreq_policy *policy, +static int longhaul_setstate(struct cpufreq_policy *policy, unsigned int table_index) { unsigned int mults_index; @@ -258,10 +259,12 @@ static void longhaul_setstate(struct cpufreq_policy *policy, /* Safety precautions */ mult = mults[mults_index & 0x1f]; if (mult == -1) - return; + return -EINVAL; + speed = calc_speed(mult); if ((speed > highest_speed) || (speed < lowest_speed)) - return; + return -EINVAL; + /* Voltage transition before frequency transition? */ if (can_scale_voltage && longhaul_index < table_index) dir = 1; @@ -269,8 +272,6 @@ static void longhaul_setstate(struct cpufreq_policy *policy, freqs.old = calc_speed(longhaul_get_cpu_mult()); freqs.new = speed; - cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE); - pr_debug("Setting to FSB:%dMHz Mult:%d.%dx (%s)\n", fsb, mult/10, mult%10, print_speed(speed/1000)); retry_loop: @@ -347,14 +348,13 @@ retry_loop: freqs.new = calc_speed(longhaul_get_cpu_mult()); /* Check if requested frequency is set. */ if (unlikely(freqs.new != speed)) { - printk(KERN_INFO PFX "Failed to set requested frequency!\n"); + pr_info("Failed to set requested frequency!\n"); /* Revision ID = 1 but processor is expecting revision key * equal to 0. Jumpers at the bottom of processor will change * multiplier and FSB, but will not change bits in Longhaul * MSR nor enable voltage scaling. */ if (!revid_errata) { - printk(KERN_INFO PFX "Enabling \"Ignore Revision ID\" " - "option.\n"); + pr_info("Enabling \"Ignore Revision ID\" option\n"); revid_errata = 1; msleep(200); goto retry_loop; @@ -364,11 +364,10 @@ retry_loop: * but it doesn't change frequency. I tried poking various * bits in northbridge registers, but without success. */ if (longhaul_flags & USE_ACPI_C3) { - printk(KERN_INFO PFX "Disabling ACPI C3 support.\n"); + pr_info("Disabling ACPI C3 support\n"); longhaul_flags &= ~USE_ACPI_C3; if (revid_errata) { - printk(KERN_INFO PFX "Disabling \"Ignore " - "Revision ID\" option.\n"); + pr_info("Disabling \"Ignore Revision ID\" option\n"); revid_errata = 0; } msleep(200); @@ -379,18 +378,19 @@ retry_loop: * RevID = 1. RevID errata will make things right. Just * to be 100% sure. */ if (longhaul_version == TYPE_LONGHAUL_V2) { - printk(KERN_INFO PFX "Switching to Longhaul ver. 1\n"); + pr_info("Switching to Longhaul ver. 1\n"); longhaul_version = TYPE_LONGHAUL_V1; msleep(200); goto retry_loop; } } - /* Report true CPU frequency */ - cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); - if (!bm_timeout) - printk(KERN_INFO PFX "Warning: Timeout while waiting for " - "idle PCI bus.\n"); + if (!bm_timeout) { + pr_info("Warning: Timeout while waiting for idle PCI bus\n"); + return -EBUSY; + } + + return 0; } /* @@ -408,10 +408,10 @@ static int guess_fsb(int mult) { int speed = cpu_khz / 1000; int i; - int speeds[] = { 666, 1000, 1333, 2000 }; + static const int speeds[] = { 666, 1000, 1333, 2000 }; int f_max, f_min; - for (i = 0; i < 4; i++) { + for (i = 0; i < ARRAY_SIZE(speeds); i++) { f_max = ((speeds[i] * mult) + 50) / 100; f_max += (ROUNDING / 2); f_min = f_max - ROUNDING; @@ -422,7 +422,7 @@ static int guess_fsb(int mult) } -static int __cpuinit longhaul_get_ranges(void) +static int longhaul_get_ranges(void) { unsigned int i, j, k = 0; unsigned int ratio; @@ -431,12 +431,12 @@ static int __cpuinit longhaul_get_ranges(void) /* Get current frequency */ mult = longhaul_get_cpu_mult(); if (mult == -1) { - printk(KERN_INFO PFX "Invalid (reserved) multiplier!\n"); + pr_info("Invalid (reserved) multiplier!\n"); return -EINVAL; } fsb = guess_fsb(mult); if (fsb == 0) { - printk(KERN_INFO PFX "Invalid (reserved) FSB!\n"); + pr_info("Invalid (reserved) FSB!\n"); return -EINVAL; } /* Get max multiplier - as we always did. @@ -466,17 +466,17 @@ static int __cpuinit longhaul_get_ranges(void) print_speed(highest_speed/1000)); if (lowest_speed == highest_speed) { - printk(KERN_INFO PFX "highestspeed == lowest, aborting.\n"); + pr_info("highestspeed == lowest, aborting\n"); return -EINVAL; } if (lowest_speed > highest_speed) { - printk(KERN_INFO PFX "nonsense! lowest (%d > %d) !\n", + pr_info("nonsense! lowest (%d > %d) !\n", lowest_speed, highest_speed); return -EINVAL; } - longhaul_table = kmalloc((numscales + 1) * sizeof(*longhaul_table), - GFP_KERNEL); + longhaul_table = kcalloc(numscales + 1, sizeof(*longhaul_table), + GFP_KERNEL); if (!longhaul_table) return -ENOMEM; @@ -526,25 +526,26 @@ static int __cpuinit longhaul_get_ranges(void) } -static void __cpuinit longhaul_setup_voltagescaling(void) +static void longhaul_setup_voltagescaling(void) { + struct cpufreq_frequency_table *freq_pos; union msr_longhaul longhaul; struct mV_pos minvid, maxvid, vid; unsigned int j, speed, pos, kHz_step, numvscales; int min_vid_speed; - rdmsrl(MSR_VIA_LONGHAUL, longhaul.val); + rdmsrq(MSR_VIA_LONGHAUL, longhaul.val); if (!(longhaul.bits.RevisionID & 1)) { - printk(KERN_INFO PFX "Voltage scaling not supported by CPU.\n"); + pr_info("Voltage scaling not supported by CPU\n"); return; } if (!longhaul.bits.VRMRev) { - printk(KERN_INFO PFX "VRM 8.5\n"); + pr_info("VRM 8.5\n"); vrm_mV_table = &vrm85_mV[0]; mV_vrm_table = &mV_vrm85[0]; } else { - printk(KERN_INFO PFX "Mobile VRM\n"); + pr_info("Mobile VRM\n"); if (cpu_model < CPU_NEHEMIAH) return; vrm_mV_table = &mobilevrm_mV[0]; @@ -555,27 +556,21 @@ static void __cpuinit longhaul_setup_voltagescaling(void) maxvid = vrm_mV_table[longhaul.bits.MaximumVID]; if (minvid.mV == 0 || maxvid.mV == 0 || minvid.mV > maxvid.mV) { - printk(KERN_INFO PFX "Bogus values Min:%d.%03d Max:%d.%03d. " - "Voltage scaling disabled.\n", - minvid.mV/1000, minvid.mV%1000, - maxvid.mV/1000, maxvid.mV%1000); + pr_info("Bogus values Min:%d.%03d Max:%d.%03d - Voltage scaling disabled\n", + minvid.mV/1000, minvid.mV%1000, + maxvid.mV/1000, maxvid.mV%1000); return; } if (minvid.mV == maxvid.mV) { - printk(KERN_INFO PFX "Claims to support voltage scaling but " - "min & max are both %d.%03d. " - "Voltage scaling disabled\n", - maxvid.mV/1000, maxvid.mV%1000); + pr_info("Claims to support voltage scaling but min & max are both %d.%03d - Voltage scaling disabled\n", + maxvid.mV/1000, maxvid.mV%1000); return; } /* How many voltage steps*/ numvscales = maxvid.pos - minvid.pos + 1; - printk(KERN_INFO PFX - "Max VID=%d.%03d " - "Min VID=%d.%03d, " - "%d possible voltage scales\n", + pr_info("Max VID=%d.%03d Min VID=%d.%03d, %d possible voltage scales\n", maxvid.mV/1000, maxvid.mV%1000, minvid.mV/1000, minvid.mV%1000, numvscales); @@ -599,56 +594,39 @@ static void __cpuinit longhaul_setup_voltagescaling(void) break; default: return; - break; } if (min_vid_speed >= highest_speed) return; /* Calculate kHz for one voltage step */ kHz_step = (highest_speed - min_vid_speed) / numvscales; - j = 0; - while (longhaul_table[j].frequency != CPUFREQ_TABLE_END) { - speed = longhaul_table[j].frequency; + cpufreq_for_each_entry_idx(freq_pos, longhaul_table, j) { + speed = freq_pos->frequency; if (speed > min_vid_speed) pos = (speed - min_vid_speed) / kHz_step + minvid.pos; else pos = minvid.pos; - longhaul_table[j].driver_data |= mV_vrm_table[pos] << 8; + freq_pos->driver_data |= mV_vrm_table[pos] << 8; vid = vrm_mV_table[mV_vrm_table[pos]]; - printk(KERN_INFO PFX "f: %d kHz, index: %d, vid: %d mV\n", - speed, j, vid.mV); - j++; + pr_info("f: %d kHz, index: %d, vid: %d mV\n", + speed, j, vid.mV); } can_scale_voltage = 1; - printk(KERN_INFO PFX "Voltage scaling enabled.\n"); -} - - -static int longhaul_verify(struct cpufreq_policy *policy) -{ - return cpufreq_frequency_table_verify(policy, longhaul_table); + pr_info("Voltage scaling enabled\n"); } static int longhaul_target(struct cpufreq_policy *policy, - unsigned int target_freq, unsigned int relation) + unsigned int table_index) { - unsigned int table_index = 0; unsigned int i; unsigned int dir = 0; u8 vid, current_vid; - - if (cpufreq_frequency_table_target(policy, longhaul_table, target_freq, - relation, &table_index)) - return -EINVAL; - - /* Don't set same frequency again */ - if (longhaul_index == table_index) - return 0; + int retval = 0; if (!can_scale_voltage) - longhaul_setstate(policy, table_index); + retval = longhaul_setstate(policy, table_index); else { /* On test system voltage transitions exceeding single * step up or down were turning motherboard off. Both @@ -663,7 +641,7 @@ static int longhaul_target(struct cpufreq_policy *policy, while (i != table_index) { vid = (longhaul_table[i].driver_data >> 8) & 0x1f; if (vid != current_vid) { - longhaul_setstate(policy, i); + retval = longhaul_setstate(policy, i); current_vid = vid; msleep(200); } @@ -672,10 +650,11 @@ static int longhaul_target(struct cpufreq_policy *policy, else i--; } - longhaul_setstate(policy, table_index); + retval = longhaul_setstate(policy, table_index); } + longhaul_index = table_index; - return 0; + return retval; } @@ -690,9 +669,9 @@ static acpi_status longhaul_walk_callback(acpi_handle obj_handle, u32 nesting_level, void *context, void **return_value) { - struct acpi_device *d; + struct acpi_device *d = acpi_fetch_acpi_dev(obj_handle); - if (acpi_bus_get_device(obj_handle, &d)) + if (!d) return 0; *return_value = acpi_driver_data(d); @@ -732,8 +711,7 @@ static int enable_arbiter_disable(void) pci_write_config_byte(dev, reg, pci_cmd); pci_read_config_byte(dev, reg, &pci_cmd); if (!(pci_cmd & 1<<7)) { - printk(KERN_ERR PFX - "Can't enable access to port 0x22.\n"); + pr_err("Can't enable access to port 0x22\n"); status = 0; } } @@ -770,8 +748,7 @@ static int longhaul_setup_southbridge(void) if (pci_cmd & 1 << 7) { pci_read_config_dword(dev, 0x88, &acpi_regs_addr); acpi_regs_addr &= 0xff00; - printk(KERN_INFO PFX "ACPI I/O at 0x%x\n", - acpi_regs_addr); + pr_info("ACPI I/O at 0x%x\n", acpi_regs_addr); } pci_dev_put(dev); @@ -780,7 +757,7 @@ static int longhaul_setup_southbridge(void) return 0; } -static int __cpuinit longhaul_cpu_init(struct cpufreq_policy *policy) +static int longhaul_cpu_init(struct cpufreq_policy *policy) { struct cpuinfo_x86 *c = &cpu_data(0); char *cpuname = NULL; @@ -798,7 +775,7 @@ static int __cpuinit longhaul_cpu_init(struct cpufreq_policy *policy) break; case 7: - switch (c->x86_mask) { + switch (c->x86_stepping) { case 0: longhaul_version = TYPE_LONGHAUL_V1; cpu_model = CPU_SAMUEL2; @@ -810,7 +787,7 @@ static int __cpuinit longhaul_cpu_init(struct cpufreq_policy *policy) break; case 1 ... 15: longhaul_version = TYPE_LONGHAUL_V2; - if (c->x86_mask < 8) { + if (c->x86_stepping < 8) { cpu_model = CPU_SAMUEL2; cpuname = "C3 'Samuel 2' [C5B]"; } else { @@ -837,7 +814,7 @@ static int __cpuinit longhaul_cpu_init(struct cpufreq_policy *policy) numscales = 32; memcpy(mults, nehemiah_mults, sizeof(nehemiah_mults)); memcpy(eblcr, nehemiah_eblcr, sizeof(nehemiah_eblcr)); - switch (c->x86_mask) { + switch (c->x86_stepping) { case 0 ... 1: cpu_model = CPU_NEHEMIAH; cpuname = "C3 'Nehemiah A' [C5XLOE]"; @@ -865,16 +842,16 @@ static int __cpuinit longhaul_cpu_init(struct cpufreq_policy *policy) longhaul_version = TYPE_LONGHAUL_V1; } - printk(KERN_INFO PFX "VIA %s CPU detected. ", cpuname); + pr_info("VIA %s CPU detected. ", cpuname); switch (longhaul_version) { case TYPE_LONGHAUL_V1: case TYPE_LONGHAUL_V2: - printk(KERN_CONT "Longhaul v%d supported.\n", longhaul_version); + pr_cont("Longhaul v%d supported\n", longhaul_version); break; case TYPE_POWERSAVER: - printk(KERN_CONT "Powersaver supported.\n"); + pr_cont("Powersaver supported\n"); break; - }; + } /* Doesn't hurt */ longhaul_setup_southbridge(); @@ -901,15 +878,14 @@ static int __cpuinit longhaul_cpu_init(struct cpufreq_policy *policy) if (!(longhaul_flags & USE_ACPI_C3 || longhaul_flags & USE_NORTHBRIDGE) && ((pr == NULL) || !(pr->flags.bm_control))) { - printk(KERN_ERR PFX - "No ACPI support. Unsupported northbridge.\n"); + pr_err("No ACPI support: Unsupported northbridge\n"); return -ENODEV; } if (longhaul_flags & USE_NORTHBRIDGE) - printk(KERN_INFO PFX "Using northbridge support.\n"); + pr_info("Using northbridge support\n"); if (longhaul_flags & USE_ACPI_C3) - printk(KERN_INFO PFX "Using ACPI support.\n"); + pr_info("Using ACPI support\n"); ret = longhaul_get_ranges(); if (ret != 0) @@ -918,42 +894,22 @@ static int __cpuinit longhaul_cpu_init(struct cpufreq_policy *policy) if ((longhaul_version != TYPE_LONGHAUL_V1) && (scale_voltage != 0)) longhaul_setup_voltagescaling(); - policy->cpuinfo.transition_latency = 200000; /* nsec */ - policy->cur = calc_speed(longhaul_get_cpu_mult()); - - ret = cpufreq_frequency_table_cpuinfo(policy, longhaul_table); - if (ret) - return ret; - - cpufreq_frequency_table_get_attr(longhaul_table, policy->cpu); + policy->transition_delay_us = 200000; /* usec */ + policy->freq_table = longhaul_table; return 0; } -static int longhaul_cpu_exit(struct cpufreq_policy *policy) -{ - cpufreq_frequency_table_put_attr(policy->cpu); - return 0; -} - -static struct freq_attr *longhaul_attr[] = { - &cpufreq_freq_attr_scaling_available_freqs, - NULL, -}; - static struct cpufreq_driver longhaul_driver = { - .verify = longhaul_verify, - .target = longhaul_target, + .verify = cpufreq_generic_frequency_table_verify, + .target_index = longhaul_target, .get = longhaul_get, .init = longhaul_cpu_init, - .exit = longhaul_cpu_exit, .name = "longhaul", - .owner = THIS_MODULE, - .attr = longhaul_attr, }; static const struct x86_cpu_id longhaul_id[] = { - { X86_VENDOR_CENTAUR, 6 }, + X86_MATCH_VENDOR_FAM(CENTAUR, 6, NULL), {} }; MODULE_DEVICE_TABLE(x86cpu, longhaul_id); @@ -966,20 +922,18 @@ static int __init longhaul_init(void) return -ENODEV; if (!enable) { - printk(KERN_ERR PFX "Option \"enable\" not set. Aborting.\n"); + pr_err("Option \"enable\" not set - Aborting\n"); return -ENODEV; } #ifdef CONFIG_SMP if (num_online_cpus() > 1) { - printk(KERN_ERR PFX "More than 1 CPU detected, " - "longhaul disabled.\n"); + pr_err("More than 1 CPU detected, longhaul disabled\n"); return -ENODEV; } #endif #ifdef CONFIG_X86_IO_APIC - if (cpu_has_apic) { - printk(KERN_ERR PFX "APIC detected. Longhaul is currently " - "broken in this configuration.\n"); + if (boot_cpu_has(X86_FEATURE_APIC)) { + pr_err("APIC detected. Longhaul is currently broken in this configuration.\n"); return -ENODEV; } #endif @@ -987,9 +941,7 @@ static int __init longhaul_init(void) case 6 ... 9: return cpufreq_register_driver(&longhaul_driver); case 10: - printk(KERN_ERR PFX "Use acpi-cpufreq driver for VIA C7\n"); - default: - ; + pr_err("Use acpi-cpufreq driver for VIA C7\n"); } return -ENODEV; @@ -1001,9 +953,20 @@ static void __exit longhaul_exit(void) struct cpufreq_policy *policy = cpufreq_cpu_get(0); int i; + if (unlikely(!policy)) + return; + for (i = 0; i < numscales; i++) { if (mults[i] == maxmult) { + struct cpufreq_freqs freqs; + + freqs.old = policy->cur; + freqs.new = longhaul_table[i].frequency; + freqs.flags = 0; + + cpufreq_freq_transition_begin(policy, &freqs); longhaul_setstate(policy, i); + cpufreq_freq_transition_end(policy, &freqs, 0); break; } } @@ -1032,7 +995,7 @@ MODULE_PARM_DESC(revid_errata, "Ignore CPU Revision ID"); module_param(enable, int, 0644); MODULE_PARM_DESC(enable, "Enable driver"); -MODULE_AUTHOR("Dave Jones <davej@redhat.com>"); +MODULE_AUTHOR("Dave Jones"); MODULE_DESCRIPTION("Longhaul driver for VIA Cyrix processors."); MODULE_LICENSE("GPL"); diff --git a/drivers/cpufreq/longhaul.h b/drivers/cpufreq/longhaul.h index e2dc436099d1..89c4cc297cb6 100644 --- a/drivers/cpufreq/longhaul.h +++ b/drivers/cpufreq/longhaul.h @@ -1,9 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * longhaul.h * (C) 2003 Dave Jones. * - * Licensed under the terms of the GNU GPL License version 2. - * * VIA-specific information */ @@ -56,7 +55,7 @@ union msr_longhaul { /* * VIA C3 Samuel 1 & Samuel 2 (stepping 0) */ -static const int __cpuinitconst samuel1_mults[16] = { +static const int samuel1_mults[16] = { -1, /* 0000 -> RESERVED */ 30, /* 0001 -> 3.0x */ 40, /* 0010 -> 4.0x */ @@ -75,7 +74,7 @@ static const int __cpuinitconst samuel1_mults[16] = { -1, /* 1111 -> RESERVED */ }; -static const int __cpuinitconst samuel1_eblcr[16] = { +static const int samuel1_eblcr[16] = { 50, /* 0000 -> RESERVED */ 30, /* 0001 -> 3.0x */ 40, /* 0010 -> 4.0x */ @@ -97,7 +96,7 @@ static const int __cpuinitconst samuel1_eblcr[16] = { /* * VIA C3 Samuel2 Stepping 1->15 */ -static const int __cpuinitconst samuel2_eblcr[16] = { +static const int samuel2_eblcr[16] = { 50, /* 0000 -> 5.0x */ 30, /* 0001 -> 3.0x */ 40, /* 0010 -> 4.0x */ @@ -119,7 +118,7 @@ static const int __cpuinitconst samuel2_eblcr[16] = { /* * VIA C3 Ezra */ -static const int __cpuinitconst ezra_mults[16] = { +static const int ezra_mults[16] = { 100, /* 0000 -> 10.0x */ 30, /* 0001 -> 3.0x */ 40, /* 0010 -> 4.0x */ @@ -138,7 +137,7 @@ static const int __cpuinitconst ezra_mults[16] = { 120, /* 1111 -> 12.0x */ }; -static const int __cpuinitconst ezra_eblcr[16] = { +static const int ezra_eblcr[16] = { 50, /* 0000 -> 5.0x */ 30, /* 0001 -> 3.0x */ 40, /* 0010 -> 4.0x */ @@ -160,7 +159,7 @@ static const int __cpuinitconst ezra_eblcr[16] = { /* * VIA C3 (Ezra-T) [C5M]. */ -static const int __cpuinitconst ezrat_mults[32] = { +static const int ezrat_mults[32] = { 100, /* 0000 -> 10.0x */ 30, /* 0001 -> 3.0x */ 40, /* 0010 -> 4.0x */ @@ -196,7 +195,7 @@ static const int __cpuinitconst ezrat_mults[32] = { -1, /* 1111 -> RESERVED (12.0x) */ }; -static const int __cpuinitconst ezrat_eblcr[32] = { +static const int ezrat_eblcr[32] = { 50, /* 0000 -> 5.0x */ 30, /* 0001 -> 3.0x */ 40, /* 0010 -> 4.0x */ @@ -235,7 +234,7 @@ static const int __cpuinitconst ezrat_eblcr[32] = { /* * VIA C3 Nehemiah */ -static const int __cpuinitconst nehemiah_mults[32] = { +static const int nehemiah_mults[32] = { 100, /* 0000 -> 10.0x */ -1, /* 0001 -> 16.0x */ 40, /* 0010 -> 4.0x */ @@ -270,7 +269,7 @@ static const int __cpuinitconst nehemiah_mults[32] = { -1, /* 1111 -> 12.0x */ }; -static const int __cpuinitconst nehemiah_eblcr[32] = { +static const int nehemiah_eblcr[32] = { 50, /* 0000 -> 5.0x */ 160, /* 0001 -> 16.0x */ 40, /* 0010 -> 4.0x */ @@ -315,7 +314,7 @@ struct mV_pos { unsigned short pos; }; -static const struct mV_pos __cpuinitconst vrm85_mV[32] = { +static const struct mV_pos vrm85_mV[32] = { {1250, 8}, {1200, 6}, {1150, 4}, {1100, 2}, {1050, 0}, {1800, 30}, {1750, 28}, {1700, 26}, {1650, 24}, {1600, 22}, {1550, 20}, {1500, 18}, @@ -326,14 +325,14 @@ static const struct mV_pos __cpuinitconst vrm85_mV[32] = { {1475, 17}, {1425, 15}, {1375, 13}, {1325, 11} }; -static const unsigned char __cpuinitconst mV_vrm85[32] = { +static const unsigned char mV_vrm85[32] = { 0x04, 0x14, 0x03, 0x13, 0x02, 0x12, 0x01, 0x11, 0x00, 0x10, 0x0f, 0x1f, 0x0e, 0x1e, 0x0d, 0x1d, 0x0c, 0x1c, 0x0b, 0x1b, 0x0a, 0x1a, 0x09, 0x19, 0x08, 0x18, 0x07, 0x17, 0x06, 0x16, 0x05, 0x15 }; -static const struct mV_pos __cpuinitconst mobilevrm_mV[32] = { +static const struct mV_pos mobilevrm_mV[32] = { {1750, 31}, {1700, 30}, {1650, 29}, {1600, 28}, {1550, 27}, {1500, 26}, {1450, 25}, {1400, 24}, {1350, 23}, {1300, 22}, {1250, 21}, {1200, 20}, @@ -344,7 +343,7 @@ static const struct mV_pos __cpuinitconst mobilevrm_mV[32] = { {675, 3}, {650, 2}, {625, 1}, {600, 0} }; -static const unsigned char __cpuinitconst mV_mobilevrm[32] = { +static const unsigned char mV_mobilevrm[32] = { 0x1f, 0x1e, 0x1d, 0x1c, 0x1b, 0x1a, 0x19, 0x18, 0x17, 0x16, 0x15, 0x14, 0x13, 0x12, 0x11, 0x10, 0x0f, 0x0e, 0x0d, 0x0c, 0x0b, 0x0a, 0x09, 0x08, diff --git a/drivers/cpufreq/longrun.c b/drivers/cpufreq/longrun.c index 8bc9f5fbbaeb..1caaec7c280b 100644 --- a/drivers/cpufreq/longrun.c +++ b/drivers/cpufreq/longrun.c @@ -1,8 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de> * - * Licensed under the terms of the GNU GPL License version 2. - * * BIG FAT DISCLAIMER: Work in progress code. Possibly *dangerous* */ @@ -33,7 +32,7 @@ static unsigned int longrun_low_freq, longrun_high_freq; * Reads the current LongRun policy by access to MSR_TMTA_LONGRUN_FLAGS * and MSR_TMTA_LONGRUN_CTRL */ -static void __cpuinit longrun_get_policy(struct cpufreq_policy *policy) +static void longrun_get_policy(struct cpufreq_policy *policy) { u32 msr_lo, msr_hi; @@ -123,19 +122,13 @@ static int longrun_set_policy(struct cpufreq_policy *policy) * Validates a new CPUFreq policy. This function has to be called with * cpufreq_driver locked. */ -static int longrun_verify_policy(struct cpufreq_policy *policy) +static int longrun_verify_policy(struct cpufreq_policy_data *policy) { if (!policy) return -EINVAL; policy->cpu = 0; - cpufreq_verify_within_limits(policy, - policy->cpuinfo.min_freq, - policy->cpuinfo.max_freq); - - if ((policy->policy != CPUFREQ_POLICY_POWERSAVE) && - (policy->policy != CPUFREQ_POLICY_PERFORMANCE)) - return -EINVAL; + cpufreq_verify_within_cpu_limits(policy); return 0; } @@ -163,7 +156,7 @@ static unsigned int longrun_get(unsigned int cpu) * TMTA rules: * performance_pctg = (target_freq - low_freq)/(high_freq - low_freq) */ -static int __cpuinit longrun_determine_freqs(unsigned int *low_freq, +static int longrun_determine_freqs(unsigned int *low_freq, unsigned int *high_freq) { u32 msr_lo, msr_hi; @@ -256,7 +249,7 @@ static int __cpuinit longrun_determine_freqs(unsigned int *low_freq, } -static int __cpuinit longrun_cpu_init(struct cpufreq_policy *policy) +static int longrun_cpu_init(struct cpufreq_policy *policy) { int result = 0; @@ -272,7 +265,6 @@ static int __cpuinit longrun_cpu_init(struct cpufreq_policy *policy) /* cpuinfo and default policy values */ policy->cpuinfo.min_freq = longrun_low_freq; policy->cpuinfo.max_freq = longrun_high_freq; - policy->cpuinfo.transition_latency = CPUFREQ_ETERNAL; longrun_get_policy(policy); return 0; @@ -286,12 +278,10 @@ static struct cpufreq_driver longrun_driver = { .get = longrun_get, .init = longrun_cpu_init, .name = "longrun", - .owner = THIS_MODULE, }; static const struct x86_cpu_id longrun_ids[] = { - { X86_VENDOR_TRANSMETA, X86_FAMILY_ANY, X86_MODEL_ANY, - X86_FEATURE_LONGRUN }, + X86_MATCH_VENDOR_FEATURE(TRANSMETA, X86_FEATURE_LONGRUN, NULL), {} }; MODULE_DEVICE_TABLE(x86cpu, longrun_ids); diff --git a/drivers/cpufreq/loongson2_cpufreq.c b/drivers/cpufreq/loongson2_cpufreq.c index bb838b985077..39a6c4315a60 100644 --- a/drivers/cpufreq/loongson2_cpufreq.c +++ b/drivers/cpufreq/loongson2_cpufreq.c @@ -3,29 +3,28 @@ * * The 2E revision of loongson processor not support this feature. * - * Copyright (C) 2006 - 2008 Lemote Inc. & Insititute of Computing Technology + * Copyright (C) 2006 - 2008 Lemote Inc. & Institute of Computing Technology * Author: Yanhua, yanh@lemote.com * * This file is subject to the terms and conditions of the GNU General Public * License. See the file "COPYING" in the main directory of this archive * for more details. */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + #include <linux/cpufreq.h> #include <linux/module.h> #include <linux/err.h> -#include <linux/sched.h> /* set_cpus_allowed() */ #include <linux/delay.h> #include <linux/platform_device.h> -#include <asm/clock.h> #include <asm/idle.h> -#include <asm/mach-loongson/loongson.h> +#include <asm/mach-loongson2ef/loongson.h> static uint nowait; -static struct clk *cpuclk; - static void (*saved_cpu_wait) (void); static int loongson2_cpu_freq_notifier(struct notifier_block *nb, @@ -44,59 +43,20 @@ static int loongson2_cpu_freq_notifier(struct notifier_block *nb, return 0; } -static unsigned int loongson2_cpufreq_get(unsigned int cpu) -{ - return clk_get_rate(cpuclk); -} - /* * Here we notify other drivers of the proposed change and the final change. */ static int loongson2_cpufreq_target(struct cpufreq_policy *policy, - unsigned int target_freq, - unsigned int relation) + unsigned int index) { - unsigned int cpu = policy->cpu; - unsigned int newstate = 0; - cpumask_t cpus_allowed; - struct cpufreq_freqs freqs; unsigned int freq; - cpus_allowed = current->cpus_allowed; - set_cpus_allowed_ptr(current, cpumask_of(cpu)); - - if (cpufreq_frequency_table_target - (policy, &loongson2_clockmod_table[0], target_freq, relation, - &newstate)) - return -EINVAL; - freq = ((cpu_clock_freq / 1000) * - loongson2_clockmod_table[newstate].driver_data) / 8; - if (freq < policy->min || freq > policy->max) - return -EINVAL; - - pr_debug("cpufreq: requested frequency %u Hz\n", target_freq * 1000); - - freqs.old = loongson2_cpufreq_get(cpu); - freqs.new = freq; - freqs.flags = 0; - - if (freqs.new == freqs.old) - return 0; - - /* notifiers */ - cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE); - - set_cpus_allowed_ptr(current, &cpus_allowed); + loongson2_clockmod_table[index].driver_data) / 8; /* setting the cpu frequency */ - clk_set_rate(cpuclk, freq); - - /* notifiers */ - cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); - - pr_debug("cpufreq: set frequency %u kHz\n", freq); + loongson2_cpu_set_rate(freq); return 0; } @@ -107,22 +67,9 @@ static int loongson2_cpufreq_cpu_init(struct cpufreq_policy *policy) unsigned long rate; int ret; - cpuclk = clk_get(NULL, "cpu_clk"); - if (IS_ERR(cpuclk)) { - printk(KERN_ERR "cpufreq: couldn't get CPU clk\n"); - return PTR_ERR(cpuclk); - } - rate = cpu_clock_freq / 1000; - if (!rate) { - clk_put(cpuclk); + if (!rate) return -EINVAL; - } - ret = clk_set_rate(cpuclk, rate); - if (ret) { - clk_put(cpuclk); - return ret; - } /* clock table init */ for (i = 2; @@ -130,44 +77,23 @@ static int loongson2_cpufreq_cpu_init(struct cpufreq_policy *policy) i++) loongson2_clockmod_table[i].frequency = (rate * i) / 8; - policy->cur = loongson2_cpufreq_get(policy->cpu); - - cpufreq_frequency_table_get_attr(&loongson2_clockmod_table[0], - policy->cpu); - - return cpufreq_frequency_table_cpuinfo(policy, - &loongson2_clockmod_table[0]); -} - -static int loongson2_cpufreq_verify(struct cpufreq_policy *policy) -{ - return cpufreq_frequency_table_verify(policy, - &loongson2_clockmod_table[0]); -} + ret = loongson2_cpu_set_rate(rate); + if (ret) + return ret; -static int loongson2_cpufreq_exit(struct cpufreq_policy *policy) -{ - clk_put(cpuclk); + cpufreq_generic_init(policy, &loongson2_clockmod_table[0], 0); return 0; } -static struct freq_attr *loongson2_table_attr[] = { - &cpufreq_freq_attr_scaling_available_freqs, - NULL, -}; - static struct cpufreq_driver loongson2_cpufreq_driver = { - .owner = THIS_MODULE, .name = "loongson2", .init = loongson2_cpufreq_cpu_init, - .verify = loongson2_cpufreq_verify, - .target = loongson2_cpufreq_target, - .get = loongson2_cpufreq_get, - .exit = loongson2_cpufreq_exit, - .attr = loongson2_table_attr, + .verify = cpufreq_generic_frequency_table_verify, + .target_index = loongson2_cpufreq_target, + .get = cpufreq_generic_get, }; -static struct platform_device_id platform_device_ids[] = { +static const struct platform_device_id platform_device_ids[] = { { .name = "loongson2_cpufreq", }, @@ -179,7 +105,6 @@ MODULE_DEVICE_TABLE(platform, platform_device_ids); static struct platform_driver platform_driver = { .driver = { .name = "loongson2_cpufreq", - .owner = THIS_MODULE, }, .id_table = platform_device_ids, }; @@ -197,9 +122,11 @@ static void loongson2_cpu_wait(void) u32 cpu_freq; spin_lock_irqsave(&loongson2_wait_lock, flags); - cpu_freq = LOONGSON_CHIPCFG0; - LOONGSON_CHIPCFG0 &= ~0x7; /* Put CPU into wait mode */ - LOONGSON_CHIPCFG0 = cpu_freq; /* Restore CPU state */ + cpu_freq = readl(LOONGSON_CHIPCFG); + /* Put CPU into wait mode */ + writel(readl(LOONGSON_CHIPCFG) & ~0x7, LOONGSON_CHIPCFG); + /* Restore CPU state */ + writel(cpu_freq, LOONGSON_CHIPCFG); spin_unlock_irqrestore(&loongson2_wait_lock, flags); local_irq_enable(); } @@ -213,14 +140,16 @@ static int __init cpufreq_init(void) if (ret) return ret; - pr_info("cpufreq: Loongson-2F CPU frequency driver.\n"); + pr_info("Loongson-2F CPU frequency driver\n"); cpufreq_register_notifier(&loongson2_cpufreq_notifier_block, CPUFREQ_TRANSITION_NOTIFIER); ret = cpufreq_register_driver(&loongson2_cpufreq_driver); - if (!ret && !nowait) { + if (ret) { + platform_driver_unregister(&platform_driver); + } else if (!nowait) { saved_cpu_wait = cpu_wait; cpu_wait = loongson2_cpu_wait; } diff --git a/drivers/cpufreq/loongson3_cpufreq.c b/drivers/cpufreq/loongson3_cpufreq.c new file mode 100644 index 000000000000..1e8715ea1b77 --- /dev/null +++ b/drivers/cpufreq/loongson3_cpufreq.c @@ -0,0 +1,389 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * CPUFreq driver for the Loongson-3 processors. + * + * All revisions of Loongson-3 processor support cpu_has_scalefreq feature. + * + * Author: Huacai Chen <chenhuacai@loongson.cn> + * Copyright (C) 2024 Loongson Technology Corporation Limited + */ +#include <linux/cpufreq.h> +#include <linux/delay.h> +#include <linux/module.h> +#include <linux/platform_device.h> +#include <linux/units.h> + +#include <asm/idle.h> +#include <asm/loongarch.h> +#include <asm/loongson.h> + +/* Message */ +union smc_message { + u32 value; + struct { + u32 id : 4; + u32 info : 4; + u32 val : 16; + u32 cmd : 6; + u32 extra : 1; + u32 complete : 1; + }; +}; + +/* Command return values */ +#define CMD_OK 0 /* No error */ +#define CMD_ERROR 1 /* Regular error */ +#define CMD_NOCMD 2 /* Command does not support */ +#define CMD_INVAL 3 /* Invalid Parameter */ + +/* Version commands */ +/* + * CMD_GET_VERSION - Get interface version + * Input: none + * Output: version + */ +#define CMD_GET_VERSION 0x1 + +/* Feature commands */ +/* + * CMD_GET_FEATURE - Get feature state + * Input: feature ID + * Output: feature flag + */ +#define CMD_GET_FEATURE 0x2 + +/* + * CMD_SET_FEATURE - Set feature state + * Input: feature ID, feature flag + * output: none + */ +#define CMD_SET_FEATURE 0x3 + +/* Feature IDs */ +#define FEATURE_SENSOR 0 +#define FEATURE_FAN 1 +#define FEATURE_DVFS 2 + +/* Sensor feature flags */ +#define FEATURE_SENSOR_ENABLE BIT(0) +#define FEATURE_SENSOR_SAMPLE BIT(1) + +/* Fan feature flags */ +#define FEATURE_FAN_ENABLE BIT(0) +#define FEATURE_FAN_AUTO BIT(1) + +/* DVFS feature flags */ +#define FEATURE_DVFS_ENABLE BIT(0) +#define FEATURE_DVFS_BOOST BIT(1) +#define FEATURE_DVFS_AUTO BIT(2) +#define FEATURE_DVFS_SINGLE_BOOST BIT(3) + +/* Sensor commands */ +/* + * CMD_GET_SENSOR_NUM - Get number of sensors + * Input: none + * Output: number + */ +#define CMD_GET_SENSOR_NUM 0x4 + +/* + * CMD_GET_SENSOR_STATUS - Get sensor status + * Input: sensor ID, type + * Output: sensor status + */ +#define CMD_GET_SENSOR_STATUS 0x5 + +/* Sensor types */ +#define SENSOR_INFO_TYPE 0 +#define SENSOR_INFO_TYPE_TEMP 1 + +/* Fan commands */ +/* + * CMD_GET_FAN_NUM - Get number of fans + * Input: none + * Output: number + */ +#define CMD_GET_FAN_NUM 0x6 + +/* + * CMD_GET_FAN_INFO - Get fan status + * Input: fan ID, type + * Output: fan info + */ +#define CMD_GET_FAN_INFO 0x7 + +/* + * CMD_SET_FAN_INFO - Set fan status + * Input: fan ID, type, value + * Output: none + */ +#define CMD_SET_FAN_INFO 0x8 + +/* Fan types */ +#define FAN_INFO_TYPE_LEVEL 0 + +/* DVFS commands */ +/* + * CMD_GET_FREQ_LEVEL_NUM - Get number of freq levels + * Input: CPU ID + * Output: number + */ +#define CMD_GET_FREQ_LEVEL_NUM 0x9 + +/* + * CMD_GET_FREQ_BOOST_LEVEL - Get the first boost level + * Input: CPU ID + * Output: number + */ +#define CMD_GET_FREQ_BOOST_LEVEL 0x10 + +/* + * CMD_GET_FREQ_LEVEL_INFO - Get freq level info + * Input: CPU ID, level ID + * Output: level info + */ +#define CMD_GET_FREQ_LEVEL_INFO 0x11 + +/* + * CMD_GET_FREQ_INFO - Get freq info + * Input: CPU ID, type + * Output: freq info + */ +#define CMD_GET_FREQ_INFO 0x12 + +/* + * CMD_SET_FREQ_INFO - Set freq info + * Input: CPU ID, type, value + * Output: none + */ +#define CMD_SET_FREQ_INFO 0x13 + +/* Freq types */ +#define FREQ_INFO_TYPE_FREQ 0 +#define FREQ_INFO_TYPE_LEVEL 1 + +#define FREQ_MAX_LEVEL 16 + +struct loongson3_freq_data { + unsigned int def_freq_level; + struct cpufreq_frequency_table table[]; +}; + +static struct mutex cpufreq_mutex[MAX_PACKAGES]; +static struct cpufreq_driver loongson3_cpufreq_driver; +static DEFINE_PER_CPU(struct loongson3_freq_data *, freq_data); + +static inline int do_service_request(u32 id, u32 info, u32 cmd, u32 val, u32 extra) +{ + int retries; + unsigned int cpu = raw_smp_processor_id(); + unsigned int package = cpu_data[cpu].package; + union smc_message msg, last; + + mutex_lock(&cpufreq_mutex[package]); + + last.value = iocsr_read32(LOONGARCH_IOCSR_SMCMBX); + if (!last.complete) { + mutex_unlock(&cpufreq_mutex[package]); + return -EPERM; + } + + msg.id = id; + msg.info = info; + msg.cmd = cmd; + msg.val = val; + msg.extra = extra; + msg.complete = 0; + + iocsr_write32(msg.value, LOONGARCH_IOCSR_SMCMBX); + iocsr_write32(iocsr_read32(LOONGARCH_IOCSR_MISC_FUNC) | IOCSR_MISC_FUNC_SOFT_INT, + LOONGARCH_IOCSR_MISC_FUNC); + + for (retries = 0; retries < 10000; retries++) { + msg.value = iocsr_read32(LOONGARCH_IOCSR_SMCMBX); + if (msg.complete) + break; + + usleep_range(8, 12); + } + + if (!msg.complete || msg.cmd != CMD_OK) { + mutex_unlock(&cpufreq_mutex[package]); + return -EPERM; + } + + mutex_unlock(&cpufreq_mutex[package]); + + return msg.val; +} + +static unsigned int loongson3_cpufreq_get(unsigned int cpu) +{ + int ret; + + ret = do_service_request(cpu, FREQ_INFO_TYPE_FREQ, CMD_GET_FREQ_INFO, 0, 0); + + return ret * KILO; +} + +static int loongson3_cpufreq_target(struct cpufreq_policy *policy, unsigned int index) +{ + int ret; + + ret = do_service_request(cpu_data[policy->cpu].core, + FREQ_INFO_TYPE_LEVEL, CMD_SET_FREQ_INFO, index, 0); + + return (ret >= 0) ? 0 : ret; +} + +static int configure_freq_table(int cpu) +{ + int i, ret, boost_level, max_level, freq_level; + struct platform_device *pdev = cpufreq_get_driver_data(); + struct loongson3_freq_data *data; + + if (per_cpu(freq_data, cpu)) + return 0; + + ret = do_service_request(cpu, 0, CMD_GET_FREQ_LEVEL_NUM, 0, 0); + if (ret < 0) + return ret; + max_level = ret; + + ret = do_service_request(cpu, 0, CMD_GET_FREQ_BOOST_LEVEL, 0, 0); + if (ret < 0) + return ret; + boost_level = ret; + + freq_level = min(max_level, FREQ_MAX_LEVEL); + data = devm_kzalloc(&pdev->dev, struct_size(data, table, freq_level + 1), GFP_KERNEL); + if (!data) + return -ENOMEM; + + data->def_freq_level = boost_level - 1; + + for (i = 0; i < freq_level; i++) { + ret = do_service_request(cpu, FREQ_INFO_TYPE_FREQ, CMD_GET_FREQ_LEVEL_INFO, i, 0); + if (ret < 0) { + devm_kfree(&pdev->dev, data); + return ret; + } + + data->table[i].frequency = ret * KILO; + data->table[i].flags = (i >= boost_level) ? CPUFREQ_BOOST_FREQ : 0; + } + + data->table[freq_level].flags = 0; + data->table[freq_level].frequency = CPUFREQ_TABLE_END; + + per_cpu(freq_data, cpu) = data; + + return 0; +} + +static int loongson3_cpufreq_cpu_init(struct cpufreq_policy *policy) +{ + int i, ret, cpu = policy->cpu; + + ret = configure_freq_table(cpu); + if (ret < 0) + return ret; + + policy->cpuinfo.transition_latency = 10000; + policy->freq_table = per_cpu(freq_data, cpu)->table; + policy->suspend_freq = policy->freq_table[per_cpu(freq_data, cpu)->def_freq_level].frequency; + cpumask_copy(policy->cpus, topology_sibling_cpumask(cpu)); + + for_each_cpu(i, policy->cpus) { + if (i != cpu) + per_cpu(freq_data, i) = per_cpu(freq_data, cpu); + } + + return 0; +} + +static void loongson3_cpufreq_cpu_exit(struct cpufreq_policy *policy) +{ + int cpu = policy->cpu; + + loongson3_cpufreq_target(policy, per_cpu(freq_data, cpu)->def_freq_level); +} + +static int loongson3_cpufreq_cpu_online(struct cpufreq_policy *policy) +{ + return 0; +} + +static int loongson3_cpufreq_cpu_offline(struct cpufreq_policy *policy) +{ + return 0; +} + +static struct cpufreq_driver loongson3_cpufreq_driver = { + .name = "loongson3", + .flags = CPUFREQ_CONST_LOOPS, + .init = loongson3_cpufreq_cpu_init, + .exit = loongson3_cpufreq_cpu_exit, + .online = loongson3_cpufreq_cpu_online, + .offline = loongson3_cpufreq_cpu_offline, + .get = loongson3_cpufreq_get, + .target_index = loongson3_cpufreq_target, + .verify = cpufreq_generic_frequency_table_verify, + .set_boost = cpufreq_boost_set_sw, + .suspend = cpufreq_generic_suspend, +}; + +static int loongson3_cpufreq_probe(struct platform_device *pdev) +{ + int i, ret; + + for (i = 0; i < MAX_PACKAGES; i++) { + ret = devm_mutex_init(&pdev->dev, &cpufreq_mutex[i]); + if (ret) + return ret; + } + + ret = do_service_request(0, 0, CMD_GET_VERSION, 0, 0); + if (ret <= 0) + return -EPERM; + + ret = do_service_request(FEATURE_DVFS, 0, CMD_SET_FEATURE, + FEATURE_DVFS_ENABLE | FEATURE_DVFS_BOOST, 0); + if (ret < 0) + return -EPERM; + + loongson3_cpufreq_driver.driver_data = pdev; + + ret = cpufreq_register_driver(&loongson3_cpufreq_driver); + if (ret) + return ret; + + pr_info("cpufreq: Loongson-3 CPU frequency driver.\n"); + + return 0; +} + +static void loongson3_cpufreq_remove(struct platform_device *pdev) +{ + cpufreq_unregister_driver(&loongson3_cpufreq_driver); +} + +static struct platform_device_id cpufreq_id_table[] = { + { "loongson3_cpufreq", }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(platform, cpufreq_id_table); + +static struct platform_driver loongson3_platform_driver = { + .driver = { + .name = "loongson3_cpufreq", + }, + .id_table = cpufreq_id_table, + .probe = loongson3_cpufreq_probe, + .remove = loongson3_cpufreq_remove, +}; +module_platform_driver(loongson3_platform_driver); + +MODULE_AUTHOR("Huacai Chen <chenhuacai@loongson.cn>"); +MODULE_DESCRIPTION("CPUFreq driver for Loongson-3 processors"); +MODULE_LICENSE("GPL"); diff --git a/drivers/cpufreq/maple-cpufreq.c b/drivers/cpufreq/maple-cpufreq.c deleted file mode 100644 index cdd62915efaf..000000000000 --- a/drivers/cpufreq/maple-cpufreq.c +++ /dev/null @@ -1,308 +0,0 @@ -/* - * Copyright (C) 2011 Dmitry Eremin-Solenikov - * Copyright (C) 2002 - 2005 Benjamin Herrenschmidt <benh@kernel.crashing.org> - * and Markus Demleitner <msdemlei@cl.uni-heidelberg.de> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This driver adds basic cpufreq support for SMU & 970FX based G5 Macs, - * that is iMac G5 and latest single CPU desktop. - */ - -#undef DEBUG - -#include <linux/module.h> -#include <linux/types.h> -#include <linux/errno.h> -#include <linux/kernel.h> -#include <linux/delay.h> -#include <linux/sched.h> -#include <linux/cpufreq.h> -#include <linux/init.h> -#include <linux/completion.h> -#include <linux/mutex.h> -#include <linux/time.h> -#include <linux/of.h> - -#define DBG(fmt...) pr_debug(fmt) - -/* see 970FX user manual */ - -#define SCOM_PCR 0x0aa001 /* PCR scom addr */ - -#define PCR_HILO_SELECT 0x80000000U /* 1 = PCR, 0 = PCRH */ -#define PCR_SPEED_FULL 0x00000000U /* 1:1 speed value */ -#define PCR_SPEED_HALF 0x00020000U /* 1:2 speed value */ -#define PCR_SPEED_QUARTER 0x00040000U /* 1:4 speed value */ -#define PCR_SPEED_MASK 0x000e0000U /* speed mask */ -#define PCR_SPEED_SHIFT 17 -#define PCR_FREQ_REQ_VALID 0x00010000U /* freq request valid */ -#define PCR_VOLT_REQ_VALID 0x00008000U /* volt request valid */ -#define PCR_TARGET_TIME_MASK 0x00006000U /* target time */ -#define PCR_STATLAT_MASK 0x00001f00U /* STATLAT value */ -#define PCR_SNOOPLAT_MASK 0x000000f0U /* SNOOPLAT value */ -#define PCR_SNOOPACC_MASK 0x0000000fU /* SNOOPACC value */ - -#define SCOM_PSR 0x408001 /* PSR scom addr */ -/* warning: PSR is a 64 bits register */ -#define PSR_CMD_RECEIVED 0x2000000000000000U /* command received */ -#define PSR_CMD_COMPLETED 0x1000000000000000U /* command completed */ -#define PSR_CUR_SPEED_MASK 0x0300000000000000U /* current speed */ -#define PSR_CUR_SPEED_SHIFT (56) - -/* - * The G5 only supports two frequencies (Quarter speed is not supported) - */ -#define CPUFREQ_HIGH 0 -#define CPUFREQ_LOW 1 - -static struct cpufreq_frequency_table maple_cpu_freqs[] = { - {CPUFREQ_HIGH, 0}, - {CPUFREQ_LOW, 0}, - {0, CPUFREQ_TABLE_END}, -}; - -static struct freq_attr *maple_cpu_freqs_attr[] = { - &cpufreq_freq_attr_scaling_available_freqs, - NULL, -}; - -/* Power mode data is an array of the 32 bits PCR values to use for - * the various frequencies, retrieved from the device-tree - */ -static int maple_pmode_cur; - -static DEFINE_MUTEX(maple_switch_mutex); - -static const u32 *maple_pmode_data; -static int maple_pmode_max; - -/* - * SCOM based frequency switching for 970FX rev3 - */ -static int maple_scom_switch_freq(int speed_mode) -{ - unsigned long flags; - int to; - - local_irq_save(flags); - - /* Clear PCR high */ - scom970_write(SCOM_PCR, 0); - /* Clear PCR low */ - scom970_write(SCOM_PCR, PCR_HILO_SELECT | 0); - /* Set PCR low */ - scom970_write(SCOM_PCR, PCR_HILO_SELECT | - maple_pmode_data[speed_mode]); - - /* Wait for completion */ - for (to = 0; to < 10; to++) { - unsigned long psr = scom970_read(SCOM_PSR); - - if ((psr & PSR_CMD_RECEIVED) == 0 && - (((psr >> PSR_CUR_SPEED_SHIFT) ^ - (maple_pmode_data[speed_mode] >> PCR_SPEED_SHIFT)) & 0x3) - == 0) - break; - if (psr & PSR_CMD_COMPLETED) - break; - udelay(100); - } - - local_irq_restore(flags); - - maple_pmode_cur = speed_mode; - ppc_proc_freq = maple_cpu_freqs[speed_mode].frequency * 1000ul; - - return 0; -} - -static int maple_scom_query_freq(void) -{ - unsigned long psr = scom970_read(SCOM_PSR); - int i; - - for (i = 0; i <= maple_pmode_max; i++) - if ((((psr >> PSR_CUR_SPEED_SHIFT) ^ - (maple_pmode_data[i] >> PCR_SPEED_SHIFT)) & 0x3) == 0) - break; - return i; -} - -/* - * Common interface to the cpufreq core - */ - -static int maple_cpufreq_verify(struct cpufreq_policy *policy) -{ - return cpufreq_frequency_table_verify(policy, maple_cpu_freqs); -} - -static int maple_cpufreq_target(struct cpufreq_policy *policy, - unsigned int target_freq, unsigned int relation) -{ - unsigned int newstate = 0; - struct cpufreq_freqs freqs; - int rc; - - if (cpufreq_frequency_table_target(policy, maple_cpu_freqs, - target_freq, relation, &newstate)) - return -EINVAL; - - if (maple_pmode_cur == newstate) - return 0; - - mutex_lock(&maple_switch_mutex); - - freqs.old = maple_cpu_freqs[maple_pmode_cur].frequency; - freqs.new = maple_cpu_freqs[newstate].frequency; - - cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE); - rc = maple_scom_switch_freq(newstate); - cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); - - mutex_unlock(&maple_switch_mutex); - - return rc; -} - -static unsigned int maple_cpufreq_get_speed(unsigned int cpu) -{ - return maple_cpu_freqs[maple_pmode_cur].frequency; -} - -static int maple_cpufreq_cpu_init(struct cpufreq_policy *policy) -{ - policy->cpuinfo.transition_latency = 12000; - policy->cur = maple_cpu_freqs[maple_scom_query_freq()].frequency; - /* secondary CPUs are tied to the primary one by the - * cpufreq core if in the secondary policy we tell it that - * it actually must be one policy together with all others. */ - cpumask_setall(policy->cpus); - cpufreq_frequency_table_get_attr(maple_cpu_freqs, policy->cpu); - - return cpufreq_frequency_table_cpuinfo(policy, - maple_cpu_freqs); -} - - -static struct cpufreq_driver maple_cpufreq_driver = { - .name = "maple", - .owner = THIS_MODULE, - .flags = CPUFREQ_CONST_LOOPS, - .init = maple_cpufreq_cpu_init, - .verify = maple_cpufreq_verify, - .target = maple_cpufreq_target, - .get = maple_cpufreq_get_speed, - .attr = maple_cpu_freqs_attr, -}; - -static int __init maple_cpufreq_init(void) -{ - struct device_node *cpus; - struct device_node *cpunode; - unsigned int psize; - unsigned long max_freq; - const u32 *valp; - u32 pvr_hi; - int rc = -ENODEV; - - /* - * Behave here like powermac driver which checks machine compatibility - * to ease merging of two drivers in future. - */ - if (!of_machine_is_compatible("Momentum,Maple") && - !of_machine_is_compatible("Momentum,Apache")) - return 0; - - cpus = of_find_node_by_path("/cpus"); - if (cpus == NULL) { - DBG("No /cpus node !\n"); - return -ENODEV; - } - - /* Get first CPU node */ - for (cpunode = NULL; - (cpunode = of_get_next_child(cpus, cpunode)) != NULL;) { - const u32 *reg = of_get_property(cpunode, "reg", NULL); - if (reg == NULL || (*reg) != 0) - continue; - if (!strcmp(cpunode->type, "cpu")) - break; - } - if (cpunode == NULL) { - printk(KERN_ERR "cpufreq: Can't find any CPU 0 node\n"); - goto bail_cpus; - } - - /* Check 970FX for now */ - /* we actually don't care on which CPU to access PVR */ - pvr_hi = PVR_VER(mfspr(SPRN_PVR)); - if (pvr_hi != 0x3c && pvr_hi != 0x44) { - printk(KERN_ERR "cpufreq: Unsupported CPU version (%x)\n", - pvr_hi); - goto bail_noprops; - } - - /* Look for the powertune data in the device-tree */ - /* - * On Maple this property is provided by PIBS in dual-processor config, - * not provided by PIBS in CPU0 config and also not provided by SLOF, - * so YMMV - */ - maple_pmode_data = of_get_property(cpunode, "power-mode-data", &psize); - if (!maple_pmode_data) { - DBG("No power-mode-data !\n"); - goto bail_noprops; - } - maple_pmode_max = psize / sizeof(u32) - 1; - - /* - * From what I see, clock-frequency is always the maximal frequency. - * The current driver can not slew sysclk yet, so we really only deal - * with powertune steps for now. We also only implement full freq and - * half freq in this version. So far, I haven't yet seen a machine - * supporting anything else. - */ - valp = of_get_property(cpunode, "clock-frequency", NULL); - if (!valp) - return -ENODEV; - max_freq = (*valp)/1000; - maple_cpu_freqs[0].frequency = max_freq; - maple_cpu_freqs[1].frequency = max_freq/2; - - /* Force apply current frequency to make sure everything is in - * sync (voltage is right for example). Firmware may leave us with - * a strange setting ... - */ - msleep(10); - maple_pmode_cur = -1; - maple_scom_switch_freq(maple_scom_query_freq()); - - printk(KERN_INFO "Registering Maple CPU frequency driver\n"); - printk(KERN_INFO "Low: %d Mhz, High: %d Mhz, Cur: %d MHz\n", - maple_cpu_freqs[1].frequency/1000, - maple_cpu_freqs[0].frequency/1000, - maple_cpu_freqs[maple_pmode_cur].frequency/1000); - - rc = cpufreq_register_driver(&maple_cpufreq_driver); - - of_node_put(cpunode); - of_node_put(cpus); - - return rc; - -bail_noprops: - of_node_put(cpunode); -bail_cpus: - of_node_put(cpus); - - return rc; -} - -module_init(maple_cpufreq_init); - - -MODULE_LICENSE("GPL"); diff --git a/drivers/cpufreq/mediatek-cpufreq-hw.c b/drivers/cpufreq/mediatek-cpufreq-hw.c new file mode 100644 index 000000000000..ae4500ab4891 --- /dev/null +++ b/drivers/cpufreq/mediatek-cpufreq-hw.c @@ -0,0 +1,442 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2020 MediaTek Inc. + */ + +#include <linux/bitfield.h> +#include <linux/cpufreq.h> +#include <linux/energy_model.h> +#include <linux/init.h> +#include <linux/iopoll.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_platform.h> +#include <linux/platform_device.h> +#include <linux/regulator/consumer.h> +#include <linux/slab.h> + +#define LUT_MAX_ENTRIES 32U +#define LUT_FREQ GENMASK(11, 0) +#define LUT_ROW_SIZE 0x4 +#define CPUFREQ_HW_STATUS BIT(0) +#define SVS_HW_STATUS BIT(1) +#define POLL_USEC 1000 +#define TIMEOUT_USEC 300000 + +#define FDVFS_FDIV_HZ (26 * 1000) + +enum { + REG_FREQ_LUT_TABLE, + REG_FREQ_ENABLE, + REG_FREQ_PERF_STATE, + REG_FREQ_HW_STATE, + REG_EM_POWER_TBL, + REG_FREQ_LATENCY, + + REG_ARRAY_SIZE, +}; + +struct mtk_cpufreq_priv { + struct device *dev; + const struct mtk_cpufreq_variant *variant; + void __iomem *fdvfs; +}; + +struct mtk_cpufreq_domain { + struct mtk_cpufreq_priv *parent; + struct cpufreq_frequency_table *table; + void __iomem *reg_bases[REG_ARRAY_SIZE]; + struct resource *res; + void __iomem *base; + int nr_opp; +}; + +struct mtk_cpufreq_variant { + int (*init)(struct mtk_cpufreq_priv *priv); + const u16 reg_offsets[REG_ARRAY_SIZE]; + const bool is_hybrid_dvfs; +}; + +static const struct mtk_cpufreq_variant cpufreq_mtk_base_variant = { + .reg_offsets = { + [REG_FREQ_LUT_TABLE] = 0x0, + [REG_FREQ_ENABLE] = 0x84, + [REG_FREQ_PERF_STATE] = 0x88, + [REG_FREQ_HW_STATE] = 0x8c, + [REG_EM_POWER_TBL] = 0x90, + [REG_FREQ_LATENCY] = 0x110, + }, +}; + +static int mtk_cpufreq_hw_mt8196_init(struct mtk_cpufreq_priv *priv) +{ + priv->fdvfs = devm_of_iomap(priv->dev, priv->dev->of_node, 0, NULL); + if (IS_ERR(priv->fdvfs)) + return dev_err_probe(priv->dev, PTR_ERR(priv->fdvfs), + "failed to get fdvfs iomem\n"); + + return 0; +} + +static const struct mtk_cpufreq_variant cpufreq_mtk_mt8196_variant = { + .init = mtk_cpufreq_hw_mt8196_init, + .reg_offsets = { + [REG_FREQ_LUT_TABLE] = 0x0, + [REG_FREQ_ENABLE] = 0x84, + [REG_FREQ_PERF_STATE] = 0x88, + [REG_FREQ_HW_STATE] = 0x8c, + [REG_EM_POWER_TBL] = 0x90, + [REG_FREQ_LATENCY] = 0x114, + }, + .is_hybrid_dvfs = true, +}; + +static int __maybe_unused +mtk_cpufreq_get_cpu_power(struct device *cpu_dev, unsigned long *uW, + unsigned long *KHz) +{ + struct mtk_cpufreq_domain *data; + struct cpufreq_policy *policy; + int i; + + policy = cpufreq_cpu_get_raw(cpu_dev->id); + if (!policy) + return -EINVAL; + + data = policy->driver_data; + + for (i = 0; i < data->nr_opp; i++) { + if (data->table[i].frequency < *KHz) + break; + } + i--; + + *KHz = data->table[i].frequency; + /* Provide micro-Watts value to the Energy Model */ + *uW = readl_relaxed(data->reg_bases[REG_EM_POWER_TBL] + + i * LUT_ROW_SIZE); + + return 0; +} + +static void mtk_cpufreq_hw_fdvfs_switch(unsigned int target_freq, + struct cpufreq_policy *policy) +{ + struct mtk_cpufreq_domain *data = policy->driver_data; + struct mtk_cpufreq_priv *priv = data->parent; + unsigned int cpu; + + target_freq = DIV_ROUND_UP(target_freq, FDVFS_FDIV_HZ); + for_each_cpu(cpu, policy->real_cpus) { + writel_relaxed(target_freq, priv->fdvfs + cpu * 4); + } +} + +static int mtk_cpufreq_hw_target_index(struct cpufreq_policy *policy, + unsigned int index) +{ + struct mtk_cpufreq_domain *data = policy->driver_data; + unsigned int target_freq; + + if (data->parent->fdvfs) { + target_freq = policy->freq_table[index].frequency; + mtk_cpufreq_hw_fdvfs_switch(target_freq, policy); + } else { + writel_relaxed(index, data->reg_bases[REG_FREQ_PERF_STATE]); + } + + return 0; +} + +static unsigned int mtk_cpufreq_hw_get(unsigned int cpu) +{ + struct mtk_cpufreq_domain *data; + struct cpufreq_policy *policy; + unsigned int index; + + policy = cpufreq_cpu_get_raw(cpu); + if (!policy) + return 0; + + data = policy->driver_data; + + index = readl_relaxed(data->reg_bases[REG_FREQ_PERF_STATE]); + index = min(index, LUT_MAX_ENTRIES - 1); + + return data->table[index].frequency; +} + +static unsigned int mtk_cpufreq_hw_fast_switch(struct cpufreq_policy *policy, + unsigned int target_freq) +{ + struct mtk_cpufreq_domain *data = policy->driver_data; + unsigned int index; + + index = cpufreq_table_find_index_dl(policy, target_freq, false); + + if (data->parent->fdvfs) + mtk_cpufreq_hw_fdvfs_switch(target_freq, policy); + else + writel_relaxed(index, data->reg_bases[REG_FREQ_PERF_STATE]); + + return policy->freq_table[index].frequency; +} + +static int mtk_cpu_create_freq_table(struct platform_device *pdev, + struct mtk_cpufreq_domain *data) +{ + struct device *dev = &pdev->dev; + u32 temp, i, freq, prev_freq = 0; + void __iomem *base_table; + + data->table = devm_kcalloc(dev, LUT_MAX_ENTRIES + 1, + sizeof(*data->table), GFP_KERNEL); + if (!data->table) + return -ENOMEM; + + base_table = data->reg_bases[REG_FREQ_LUT_TABLE]; + + for (i = 0; i < LUT_MAX_ENTRIES; i++) { + temp = readl_relaxed(base_table + (i * LUT_ROW_SIZE)); + freq = FIELD_GET(LUT_FREQ, temp) * 1000; + + if (freq == prev_freq) + break; + + data->table[i].frequency = freq; + + dev_dbg(dev, "index=%d freq=%d\n", i, data->table[i].frequency); + + prev_freq = freq; + } + + data->table[i].frequency = CPUFREQ_TABLE_END; + data->nr_opp = i; + + return 0; +} + +static int mtk_cpu_resources_init(struct platform_device *pdev, + struct cpufreq_policy *policy, + struct mtk_cpufreq_priv *priv) +{ + struct mtk_cpufreq_domain *data; + struct device *dev = &pdev->dev; + struct resource *res; + struct of_phandle_args args; + void __iomem *base; + int ret, i; + int index; + + data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + ret = of_perf_domain_get_sharing_cpumask(policy->cpu, "performance-domains", + "#performance-domain-cells", + policy->cpus, &args); + if (ret < 0) + return ret; + + index = args.args[0]; + of_node_put(args.np); + + /* + * In a cpufreq with hybrid DVFS, such as the MT8196, the first declared + * register range is for FDVFS, followed by the frequency domain MMIOs. + */ + if (priv->variant->is_hybrid_dvfs) + index++; + + data->parent = priv; + + res = platform_get_resource(pdev, IORESOURCE_MEM, index); + if (!res) { + dev_err(dev, "failed to get mem resource %d\n", index); + return -ENODEV; + } + + if (!request_mem_region(res->start, resource_size(res), res->name)) { + dev_err(dev, "failed to request resource %pR\n", res); + return -EBUSY; + } + + base = ioremap(res->start, resource_size(res)); + if (!base) { + dev_err(dev, "failed to map resource %pR\n", res); + ret = -ENOMEM; + goto release_region; + } + + data->base = base; + data->res = res; + + for (i = REG_FREQ_LUT_TABLE; i < REG_ARRAY_SIZE; i++) + data->reg_bases[i] = base + priv->variant->reg_offsets[i]; + + ret = mtk_cpu_create_freq_table(pdev, data); + if (ret) { + dev_info(dev, "Domain-%d failed to create freq table\n", index); + return ret; + } + + policy->freq_table = data->table; + policy->driver_data = data; + + return 0; +release_region: + release_mem_region(res->start, resource_size(res)); + return ret; +} + +static int mtk_cpufreq_hw_cpu_init(struct cpufreq_policy *policy) +{ + struct platform_device *pdev = cpufreq_get_driver_data(); + int sig, pwr_hw = CPUFREQ_HW_STATUS | SVS_HW_STATUS; + struct mtk_cpufreq_domain *data; + unsigned int latency; + int ret; + + /* Get the bases of cpufreq for domains */ + ret = mtk_cpu_resources_init(pdev, policy, platform_get_drvdata(pdev)); + if (ret) { + dev_info(&pdev->dev, "CPUFreq resource init failed\n"); + return ret; + } + + data = policy->driver_data; + + latency = readl_relaxed(data->reg_bases[REG_FREQ_LATENCY]) * 1000; + if (!latency) + latency = CPUFREQ_DEFAULT_TRANSITION_LATENCY_NS; + + policy->cpuinfo.transition_latency = latency; + policy->fast_switch_possible = true; + + /* HW should be in enabled state to proceed now */ + writel_relaxed(0x1, data->reg_bases[REG_FREQ_ENABLE]); + if (readl_poll_timeout(data->reg_bases[REG_FREQ_HW_STATE], sig, + (sig & pwr_hw) == pwr_hw, POLL_USEC, + TIMEOUT_USEC)) { + if (!(sig & CPUFREQ_HW_STATUS)) { + pr_info("cpufreq hardware of CPU%d is not enabled\n", + policy->cpu); + return -ENODEV; + } + + pr_info("SVS of CPU%d is not enabled\n", policy->cpu); + } + + return 0; +} + +static void mtk_cpufreq_hw_cpu_exit(struct cpufreq_policy *policy) +{ + struct mtk_cpufreq_domain *data = policy->driver_data; + struct resource *res = data->res; + void __iomem *base = data->base; + + /* HW should be in paused state now */ + writel_relaxed(0x0, data->reg_bases[REG_FREQ_ENABLE]); + iounmap(base); + release_mem_region(res->start, resource_size(res)); +} + +static void mtk_cpufreq_register_em(struct cpufreq_policy *policy) +{ + struct em_data_callback em_cb = EM_DATA_CB(mtk_cpufreq_get_cpu_power); + struct mtk_cpufreq_domain *data = policy->driver_data; + + em_dev_register_perf_domain(get_cpu_device(policy->cpu), data->nr_opp, + &em_cb, policy->cpus, true); +} + +static struct cpufreq_driver cpufreq_mtk_hw_driver = { + .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK | + CPUFREQ_HAVE_GOVERNOR_PER_POLICY | + CPUFREQ_IS_COOLING_DEV, + .verify = cpufreq_generic_frequency_table_verify, + .target_index = mtk_cpufreq_hw_target_index, + .get = mtk_cpufreq_hw_get, + .init = mtk_cpufreq_hw_cpu_init, + .exit = mtk_cpufreq_hw_cpu_exit, + .register_em = mtk_cpufreq_register_em, + .fast_switch = mtk_cpufreq_hw_fast_switch, + .name = "mtk-cpufreq-hw", +}; + +static int mtk_cpufreq_hw_driver_probe(struct platform_device *pdev) +{ + struct mtk_cpufreq_priv *priv; + const void *data; + int ret, cpu; + struct device *cpu_dev; + struct regulator *cpu_reg; + + /* Make sure that all CPU supplies are available before proceeding. */ + for_each_present_cpu(cpu) { + cpu_dev = get_cpu_device(cpu); + if (!cpu_dev) + return dev_err_probe(&pdev->dev, -EPROBE_DEFER, + "Failed to get cpu%d device\n", cpu); + + cpu_reg = devm_regulator_get(cpu_dev, "cpu"); + if (IS_ERR(cpu_reg)) + return dev_err_probe(&pdev->dev, PTR_ERR(cpu_reg), + "CPU%d regulator get failed\n", cpu); + } + + + data = of_device_get_match_data(&pdev->dev); + if (!data) + return -EINVAL; + + priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + priv->variant = data; + priv->dev = &pdev->dev; + + if (priv->variant->init) { + ret = priv->variant->init(priv); + if (ret) + return ret; + } + + platform_set_drvdata(pdev, priv); + cpufreq_mtk_hw_driver.driver_data = pdev; + + ret = cpufreq_register_driver(&cpufreq_mtk_hw_driver); + if (ret) + dev_err(&pdev->dev, "CPUFreq HW driver failed to register\n"); + + return ret; +} + +static void mtk_cpufreq_hw_driver_remove(struct platform_device *pdev) +{ + cpufreq_unregister_driver(&cpufreq_mtk_hw_driver); +} + +static const struct of_device_id mtk_cpufreq_hw_match[] = { + { .compatible = "mediatek,cpufreq-hw", .data = &cpufreq_mtk_base_variant }, + { .compatible = "mediatek,mt8196-cpufreq-hw", .data = &cpufreq_mtk_mt8196_variant }, + {} +}; +MODULE_DEVICE_TABLE(of, mtk_cpufreq_hw_match); + +static struct platform_driver mtk_cpufreq_hw_driver = { + .probe = mtk_cpufreq_hw_driver_probe, + .remove = mtk_cpufreq_hw_driver_remove, + .driver = { + .name = "mtk-cpufreq-hw", + .of_match_table = mtk_cpufreq_hw_match, + }, +}; +module_platform_driver(mtk_cpufreq_hw_driver); + +MODULE_AUTHOR("Hector Yuan <hector.yuan@mediatek.com>"); +MODULE_DESCRIPTION("Mediatek cpufreq-hw driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/cpufreq/mediatek-cpufreq.c b/drivers/cpufreq/mediatek-cpufreq.c new file mode 100644 index 000000000000..052ca7cd2f4f --- /dev/null +++ b/drivers/cpufreq/mediatek-cpufreq.c @@ -0,0 +1,807 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2015 Linaro Ltd. + * Author: Pi-Cheng Chen <pi-cheng.chen@linaro.org> + */ + +#include <linux/clk.h> +#include <linux/cpu.h> +#include <linux/cpufreq.h> +#include <linux/cpumask.h> +#include <linux/minmax.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_platform.h> +#include <linux/platform_device.h> +#include <linux/pm_opp.h> +#include <linux/regulator/consumer.h> + +struct mtk_cpufreq_platform_data { + int min_volt_shift; + int max_volt_shift; + int proc_max_volt; + int sram_min_volt; + int sram_max_volt; + bool ccifreq_supported; +}; + +/* + * The struct mtk_cpu_dvfs_info holds necessary information for doing CPU DVFS + * on each CPU power/clock domain of Mediatek SoCs. Each CPU cluster in + * Mediatek SoCs has two voltage inputs, Vproc and Vsram. In some cases the two + * voltage inputs need to be controlled under a hardware limitation: + * 100mV < Vsram - Vproc < 200mV + * + * When scaling the clock frequency of a CPU clock domain, the clock source + * needs to be switched to another stable PLL clock temporarily until + * the original PLL becomes stable at target frequency. + */ +struct mtk_cpu_dvfs_info { + struct cpumask cpus; + struct device *cpu_dev; + struct device *cci_dev; + struct regulator *proc_reg; + struct regulator *sram_reg; + struct clk *cpu_clk; + struct clk *inter_clk; + struct list_head list_head; + int intermediate_voltage; + bool need_voltage_tracking; + int vproc_on_boot; + int pre_vproc; + /* Avoid race condition for regulators between notify and policy */ + struct mutex reg_lock; + struct notifier_block opp_nb; + unsigned int opp_cpu; + unsigned long current_freq; + const struct mtk_cpufreq_platform_data *soc_data; + int vtrack_max; + bool ccifreq_bound; +}; + +static struct platform_device *cpufreq_pdev; + +static LIST_HEAD(dvfs_info_list); + +static struct mtk_cpu_dvfs_info *mtk_cpu_dvfs_info_lookup(int cpu) +{ + struct mtk_cpu_dvfs_info *info; + + list_for_each_entry(info, &dvfs_info_list, list_head) { + if (cpumask_test_cpu(cpu, &info->cpus)) + return info; + } + + return NULL; +} + +static int mtk_cpufreq_voltage_tracking(struct mtk_cpu_dvfs_info *info, + int new_vproc) +{ + const struct mtk_cpufreq_platform_data *soc_data = info->soc_data; + struct regulator *proc_reg = info->proc_reg; + struct regulator *sram_reg = info->sram_reg; + int pre_vproc, pre_vsram, new_vsram, vsram, vproc, ret; + int retry = info->vtrack_max; + + pre_vproc = regulator_get_voltage(proc_reg); + if (pre_vproc < 0) { + dev_err(info->cpu_dev, + "invalid Vproc value: %d\n", pre_vproc); + return pre_vproc; + } + + pre_vsram = regulator_get_voltage(sram_reg); + if (pre_vsram < 0) { + dev_err(info->cpu_dev, "invalid Vsram value: %d\n", pre_vsram); + return pre_vsram; + } + + new_vsram = clamp(new_vproc + soc_data->min_volt_shift, + soc_data->sram_min_volt, soc_data->sram_max_volt); + + do { + if (pre_vproc <= new_vproc) { + vsram = clamp(pre_vproc + soc_data->max_volt_shift, + soc_data->sram_min_volt, new_vsram); + ret = regulator_set_voltage(sram_reg, vsram, + soc_data->sram_max_volt); + + if (ret) + return ret; + + if (vsram == soc_data->sram_max_volt || + new_vsram == soc_data->sram_min_volt) + vproc = new_vproc; + else + vproc = vsram - soc_data->min_volt_shift; + + ret = regulator_set_voltage(proc_reg, vproc, + soc_data->proc_max_volt); + if (ret) { + regulator_set_voltage(sram_reg, pre_vsram, + soc_data->sram_max_volt); + return ret; + } + } else { + vproc = max(new_vproc, + pre_vsram - soc_data->max_volt_shift); + ret = regulator_set_voltage(proc_reg, vproc, + soc_data->proc_max_volt); + if (ret) + return ret; + + if (vproc == new_vproc) + vsram = new_vsram; + else + vsram = max(new_vsram, + vproc + soc_data->min_volt_shift); + + ret = regulator_set_voltage(sram_reg, vsram, + soc_data->sram_max_volt); + if (ret) { + regulator_set_voltage(proc_reg, pre_vproc, + soc_data->proc_max_volt); + return ret; + } + } + + pre_vproc = vproc; + pre_vsram = vsram; + + if (--retry < 0) { + dev_err(info->cpu_dev, + "over loop count, failed to set voltage\n"); + return -EINVAL; + } + } while (vproc != new_vproc || vsram != new_vsram); + + return 0; +} + +static int mtk_cpufreq_set_voltage(struct mtk_cpu_dvfs_info *info, int vproc) +{ + const struct mtk_cpufreq_platform_data *soc_data = info->soc_data; + int ret; + + if (info->need_voltage_tracking) + ret = mtk_cpufreq_voltage_tracking(info, vproc); + else + ret = regulator_set_voltage(info->proc_reg, vproc, + soc_data->proc_max_volt); + if (!ret) + info->pre_vproc = vproc; + + return ret; +} + +static bool is_ccifreq_ready(struct mtk_cpu_dvfs_info *info) +{ + struct device_link *sup_link; + + if (info->ccifreq_bound) + return true; + + sup_link = device_link_add(info->cpu_dev, info->cci_dev, + DL_FLAG_AUTOREMOVE_CONSUMER); + if (!sup_link) { + dev_err(info->cpu_dev, "cpu%d: sup_link is NULL\n", info->opp_cpu); + return false; + } + + if (sup_link->supplier->links.status != DL_DEV_DRIVER_BOUND) + return false; + + info->ccifreq_bound = true; + + return true; +} + +static int mtk_cpufreq_set_target(struct cpufreq_policy *policy, + unsigned int index) +{ + struct cpufreq_frequency_table *freq_table = policy->freq_table; + struct clk *cpu_clk = policy->clk; + struct clk *armpll = clk_get_parent(cpu_clk); + struct mtk_cpu_dvfs_info *info = policy->driver_data; + struct device *cpu_dev = info->cpu_dev; + struct dev_pm_opp *opp; + long freq_hz, pre_freq_hz; + int vproc, pre_vproc, inter_vproc, target_vproc, ret; + + inter_vproc = info->intermediate_voltage; + + pre_freq_hz = clk_get_rate(cpu_clk); + + mutex_lock(&info->reg_lock); + + if (unlikely(info->pre_vproc <= 0)) + pre_vproc = regulator_get_voltage(info->proc_reg); + else + pre_vproc = info->pre_vproc; + + if (pre_vproc < 0) { + dev_err(cpu_dev, "invalid Vproc value: %d\n", pre_vproc); + ret = pre_vproc; + goto out; + } + + freq_hz = freq_table[index].frequency * 1000; + + opp = dev_pm_opp_find_freq_ceil(cpu_dev, &freq_hz); + if (IS_ERR(opp)) { + dev_err(cpu_dev, "cpu%d: failed to find OPP for %ld\n", + policy->cpu, freq_hz); + ret = PTR_ERR(opp); + goto out; + } + vproc = dev_pm_opp_get_voltage(opp); + dev_pm_opp_put(opp); + + /* + * If MediaTek cci is supported but is not ready, we will use the value + * of max(target cpu voltage, booting voltage) to prevent high freqeuncy + * low voltage crash. + */ + if (info->soc_data->ccifreq_supported && !is_ccifreq_ready(info)) + vproc = max(vproc, info->vproc_on_boot); + + /* + * If the new voltage or the intermediate voltage is higher than the + * current voltage, scale up voltage first. + */ + target_vproc = max(inter_vproc, vproc); + if (pre_vproc <= target_vproc) { + ret = mtk_cpufreq_set_voltage(info, target_vproc); + if (ret) { + dev_err(cpu_dev, + "cpu%d: failed to scale up voltage!\n", policy->cpu); + mtk_cpufreq_set_voltage(info, pre_vproc); + goto out; + } + } + + /* Reparent the CPU clock to intermediate clock. */ + ret = clk_set_parent(cpu_clk, info->inter_clk); + if (ret) { + dev_err(cpu_dev, + "cpu%d: failed to re-parent cpu clock!\n", policy->cpu); + mtk_cpufreq_set_voltage(info, pre_vproc); + goto out; + } + + /* Set the original PLL to target rate. */ + ret = clk_set_rate(armpll, freq_hz); + if (ret) { + dev_err(cpu_dev, + "cpu%d: failed to scale cpu clock rate!\n", policy->cpu); + clk_set_parent(cpu_clk, armpll); + mtk_cpufreq_set_voltage(info, pre_vproc); + goto out; + } + + /* Set parent of CPU clock back to the original PLL. */ + ret = clk_set_parent(cpu_clk, armpll); + if (ret) { + dev_err(cpu_dev, + "cpu%d: failed to re-parent cpu clock!\n", policy->cpu); + mtk_cpufreq_set_voltage(info, inter_vproc); + goto out; + } + + /* + * If the new voltage is lower than the intermediate voltage or the + * original voltage, scale down to the new voltage. + */ + if (vproc < inter_vproc || vproc < pre_vproc) { + ret = mtk_cpufreq_set_voltage(info, vproc); + if (ret) { + dev_err(cpu_dev, + "cpu%d: failed to scale down voltage!\n", policy->cpu); + clk_set_parent(cpu_clk, info->inter_clk); + clk_set_rate(armpll, pre_freq_hz); + clk_set_parent(cpu_clk, armpll); + goto out; + } + } + + info->current_freq = freq_hz; + +out: + mutex_unlock(&info->reg_lock); + + return ret; +} + +static int mtk_cpufreq_opp_notifier(struct notifier_block *nb, + unsigned long event, void *data) +{ + struct dev_pm_opp *opp = data; + struct dev_pm_opp *new_opp; + struct mtk_cpu_dvfs_info *info; + unsigned long freq, volt; + int ret = 0; + + info = container_of(nb, struct mtk_cpu_dvfs_info, opp_nb); + + if (event == OPP_EVENT_ADJUST_VOLTAGE) { + freq = dev_pm_opp_get_freq(opp); + + mutex_lock(&info->reg_lock); + if (info->current_freq == freq) { + volt = dev_pm_opp_get_voltage(opp); + ret = mtk_cpufreq_set_voltage(info, volt); + if (ret) + dev_err(info->cpu_dev, + "failed to scale voltage: %d\n", ret); + } + mutex_unlock(&info->reg_lock); + } else if (event == OPP_EVENT_DISABLE) { + freq = dev_pm_opp_get_freq(opp); + + /* case of current opp item is disabled */ + if (info->current_freq == freq) { + freq = 1; + new_opp = dev_pm_opp_find_freq_ceil(info->cpu_dev, + &freq); + if (IS_ERR(new_opp)) { + dev_err(info->cpu_dev, + "all opp items are disabled\n"); + ret = PTR_ERR(new_opp); + return notifier_from_errno(ret); + } + + dev_pm_opp_put(new_opp); + + struct cpufreq_policy *policy __free(put_cpufreq_policy) + = cpufreq_cpu_get(info->opp_cpu); + if (policy) + cpufreq_driver_target(policy, freq / 1000, + CPUFREQ_RELATION_L); + } + } + + return notifier_from_errno(ret); +} + +static struct device *of_get_cci(struct device *cpu_dev) +{ + struct device_node *np; + struct platform_device *pdev; + + np = of_parse_phandle(cpu_dev->of_node, "mediatek,cci", 0); + if (!np) + return ERR_PTR(-ENODEV); + + pdev = of_find_device_by_node(np); + of_node_put(np); + if (!pdev) + return ERR_PTR(-ENODEV); + + return &pdev->dev; +} + +static int mtk_cpu_dvfs_info_init(struct mtk_cpu_dvfs_info *info, int cpu) +{ + struct device *cpu_dev; + struct dev_pm_opp *opp; + unsigned long rate; + int ret; + + cpu_dev = get_cpu_device(cpu); + if (!cpu_dev) + return dev_err_probe(cpu_dev, -ENODEV, "failed to get cpu%d device\n", cpu); + info->cpu_dev = cpu_dev; + + info->ccifreq_bound = false; + if (info->soc_data->ccifreq_supported) { + info->cci_dev = of_get_cci(info->cpu_dev); + if (IS_ERR(info->cci_dev)) + return dev_err_probe(cpu_dev, PTR_ERR(info->cci_dev), + "cpu%d: failed to get cci device\n", + cpu); + } + + info->cpu_clk = clk_get(cpu_dev, "cpu"); + if (IS_ERR(info->cpu_clk)) { + ret = PTR_ERR(info->cpu_clk); + dev_err_probe(cpu_dev, ret, "cpu%d: failed to get cpu clk\n", cpu); + goto out_put_cci_dev; + } + + info->inter_clk = clk_get(cpu_dev, "intermediate"); + if (IS_ERR(info->inter_clk)) { + ret = PTR_ERR(info->inter_clk); + dev_err_probe(cpu_dev, ret, + "cpu%d: failed to get intermediate clk\n", cpu); + goto out_free_mux_clock; + } + + info->proc_reg = regulator_get_optional(cpu_dev, "proc"); + if (IS_ERR(info->proc_reg)) { + ret = PTR_ERR(info->proc_reg); + dev_err_probe(cpu_dev, ret, + "cpu%d: failed to get proc regulator\n", cpu); + goto out_free_inter_clock; + } + + ret = regulator_enable(info->proc_reg); + if (ret) { + dev_err_probe(cpu_dev, ret, "cpu%d: failed to enable vproc\n", cpu); + goto out_free_proc_reg; + } + + /* Both presence and absence of sram regulator are valid cases. */ + info->sram_reg = regulator_get_optional(cpu_dev, "sram"); + if (IS_ERR(info->sram_reg)) { + ret = PTR_ERR(info->sram_reg); + if (ret == -EPROBE_DEFER) { + dev_err_probe(cpu_dev, ret, + "cpu%d: Failed to get sram regulator\n", cpu); + goto out_disable_proc_reg; + } + + info->sram_reg = NULL; + } else { + ret = regulator_enable(info->sram_reg); + if (ret) { + dev_err_probe(cpu_dev, ret, "cpu%d: failed to enable vsram\n", cpu); + goto out_free_sram_reg; + } + } + + /* Get OPP-sharing information from "operating-points-v2" bindings */ + ret = dev_pm_opp_of_get_sharing_cpus(cpu_dev, &info->cpus); + if (ret) { + dev_err_probe(cpu_dev, ret, + "cpu%d: failed to get OPP-sharing information\n", cpu); + goto out_disable_sram_reg; + } + + ret = dev_pm_opp_of_cpumask_add_table(&info->cpus); + if (ret) { + dev_err_probe(cpu_dev, ret, "cpu%d: no OPP table\n", cpu); + goto out_disable_sram_reg; + } + + ret = clk_prepare_enable(info->cpu_clk); + if (ret) { + dev_err_probe(cpu_dev, ret, "cpu%d: failed to enable cpu clk\n", cpu); + goto out_free_opp_table; + } + + ret = clk_prepare_enable(info->inter_clk); + if (ret) { + dev_err_probe(cpu_dev, ret, "cpu%d: failed to enable inter clk\n", cpu); + goto out_disable_mux_clock; + } + + if (info->soc_data->ccifreq_supported) { + info->vproc_on_boot = regulator_get_voltage(info->proc_reg); + if (info->vproc_on_boot < 0) { + ret = dev_err_probe(info->cpu_dev, info->vproc_on_boot, + "invalid Vproc value\n"); + goto out_disable_inter_clock; + } + } + + /* Search a safe voltage for intermediate frequency. */ + rate = clk_get_rate(info->inter_clk); + opp = dev_pm_opp_find_freq_ceil(cpu_dev, &rate); + if (IS_ERR(opp)) { + ret = dev_err_probe(cpu_dev, PTR_ERR(opp), + "cpu%d: failed to get intermediate opp\n", cpu); + goto out_disable_inter_clock; + } + info->intermediate_voltage = dev_pm_opp_get_voltage(opp); + dev_pm_opp_put(opp); + + mutex_init(&info->reg_lock); + info->current_freq = clk_get_rate(info->cpu_clk); + + info->opp_cpu = cpu; + info->opp_nb.notifier_call = mtk_cpufreq_opp_notifier; + ret = dev_pm_opp_register_notifier(cpu_dev, &info->opp_nb); + if (ret) { + dev_err_probe(cpu_dev, ret, "cpu%d: failed to register opp notifier\n", cpu); + goto out_disable_inter_clock; + } + + /* + * If SRAM regulator is present, software "voltage tracking" is needed + * for this CPU power domain. + */ + info->need_voltage_tracking = (info->sram_reg != NULL); + + /* + * We assume min voltage is 0 and tracking target voltage using + * min_volt_shift for each iteration. + * The vtrack_max is 3 times of expeted iteration count. + */ + info->vtrack_max = 3 * DIV_ROUND_UP(max(info->soc_data->sram_max_volt, + info->soc_data->proc_max_volt), + info->soc_data->min_volt_shift); + + return 0; + +out_disable_inter_clock: + clk_disable_unprepare(info->inter_clk); + +out_disable_mux_clock: + clk_disable_unprepare(info->cpu_clk); + +out_free_opp_table: + dev_pm_opp_of_cpumask_remove_table(&info->cpus); + +out_disable_sram_reg: + if (info->sram_reg) + regulator_disable(info->sram_reg); + +out_free_sram_reg: + if (info->sram_reg) + regulator_put(info->sram_reg); + +out_disable_proc_reg: + regulator_disable(info->proc_reg); + +out_free_proc_reg: + regulator_put(info->proc_reg); + +out_free_inter_clock: + clk_put(info->inter_clk); + +out_free_mux_clock: + clk_put(info->cpu_clk); + +out_put_cci_dev: + if (info->soc_data->ccifreq_supported) + put_device(info->cci_dev); + + return ret; +} + +static void mtk_cpu_dvfs_info_release(struct mtk_cpu_dvfs_info *info) +{ + regulator_disable(info->proc_reg); + regulator_put(info->proc_reg); + if (info->sram_reg) { + regulator_disable(info->sram_reg); + regulator_put(info->sram_reg); + } + clk_disable_unprepare(info->cpu_clk); + clk_put(info->cpu_clk); + clk_disable_unprepare(info->inter_clk); + clk_put(info->inter_clk); + dev_pm_opp_of_cpumask_remove_table(&info->cpus); + dev_pm_opp_unregister_notifier(info->cpu_dev, &info->opp_nb); + if (info->soc_data->ccifreq_supported) + put_device(info->cci_dev); +} + +static int mtk_cpufreq_init(struct cpufreq_policy *policy) +{ + struct mtk_cpu_dvfs_info *info; + struct cpufreq_frequency_table *freq_table; + int ret; + + info = mtk_cpu_dvfs_info_lookup(policy->cpu); + if (!info) { + pr_err("dvfs info for cpu%d is not initialized.\n", + policy->cpu); + return -EINVAL; + } + + ret = dev_pm_opp_init_cpufreq_table(info->cpu_dev, &freq_table); + if (ret) { + dev_err(info->cpu_dev, + "failed to init cpufreq table for cpu%d: %d\n", + policy->cpu, ret); + return ret; + } + + cpumask_copy(policy->cpus, &info->cpus); + policy->freq_table = freq_table; + policy->driver_data = info; + policy->clk = info->cpu_clk; + + return 0; +} + +static void mtk_cpufreq_exit(struct cpufreq_policy *policy) +{ + struct mtk_cpu_dvfs_info *info = policy->driver_data; + + dev_pm_opp_free_cpufreq_table(info->cpu_dev, &policy->freq_table); +} + +static struct cpufreq_driver mtk_cpufreq_driver = { + .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK | + CPUFREQ_HAVE_GOVERNOR_PER_POLICY | + CPUFREQ_IS_COOLING_DEV, + .verify = cpufreq_generic_frequency_table_verify, + .target_index = mtk_cpufreq_set_target, + .get = cpufreq_generic_get, + .init = mtk_cpufreq_init, + .exit = mtk_cpufreq_exit, + .register_em = cpufreq_register_em_with_opp, + .name = "mtk-cpufreq", +}; + +static int mtk_cpufreq_probe(struct platform_device *pdev) +{ + const struct mtk_cpufreq_platform_data *data; + struct mtk_cpu_dvfs_info *info, *tmp; + int cpu, ret; + + data = dev_get_platdata(&pdev->dev); + if (!data) + return dev_err_probe(&pdev->dev, -ENODEV, + "failed to get mtk cpufreq platform data\n"); + + for_each_present_cpu(cpu) { + info = mtk_cpu_dvfs_info_lookup(cpu); + if (info) + continue; + + info = devm_kzalloc(&pdev->dev, sizeof(*info), GFP_KERNEL); + if (!info) { + ret = dev_err_probe(&pdev->dev, -ENOMEM, + "Failed to allocate dvfs_info\n"); + goto release_dvfs_info_list; + } + + info->soc_data = data; + ret = mtk_cpu_dvfs_info_init(info, cpu); + if (ret) + goto release_dvfs_info_list; + + list_add(&info->list_head, &dvfs_info_list); + } + + ret = cpufreq_register_driver(&mtk_cpufreq_driver); + if (ret) { + dev_err_probe(&pdev->dev, ret, "failed to register mtk cpufreq driver\n"); + goto release_dvfs_info_list; + } + + return 0; + +release_dvfs_info_list: + list_for_each_entry_safe(info, tmp, &dvfs_info_list, list_head) { + mtk_cpu_dvfs_info_release(info); + list_del(&info->list_head); + } + + return ret; +} + +static struct platform_driver mtk_cpufreq_platdrv = { + .driver = { + .name = "mtk-cpufreq", + }, + .probe = mtk_cpufreq_probe, +}; + +static const struct mtk_cpufreq_platform_data mt2701_platform_data = { + .min_volt_shift = 100000, + .max_volt_shift = 200000, + .proc_max_volt = 1150000, + .sram_min_volt = 0, + .sram_max_volt = 1150000, + .ccifreq_supported = false, +}; + +static const struct mtk_cpufreq_platform_data mt7622_platform_data = { + .min_volt_shift = 100000, + .max_volt_shift = 200000, + .proc_max_volt = 1350000, + .sram_min_volt = 0, + .sram_max_volt = 1350000, + .ccifreq_supported = false, +}; + +static const struct mtk_cpufreq_platform_data mt7623_platform_data = { + .min_volt_shift = 100000, + .max_volt_shift = 200000, + .proc_max_volt = 1300000, + .ccifreq_supported = false, +}; + +static const struct mtk_cpufreq_platform_data mt7988_platform_data = { + .min_volt_shift = 100000, + .max_volt_shift = 200000, + .proc_max_volt = 900000, + .sram_min_volt = 0, + .sram_max_volt = 1150000, + .ccifreq_supported = true, +}; + +static const struct mtk_cpufreq_platform_data mt8183_platform_data = { + .min_volt_shift = 100000, + .max_volt_shift = 200000, + .proc_max_volt = 1150000, + .sram_min_volt = 0, + .sram_max_volt = 1150000, + .ccifreq_supported = true, +}; + +static const struct mtk_cpufreq_platform_data mt8186_platform_data = { + .min_volt_shift = 100000, + .max_volt_shift = 250000, + .proc_max_volt = 1118750, + .sram_min_volt = 850000, + .sram_max_volt = 1118750, + .ccifreq_supported = true, +}; + +static const struct mtk_cpufreq_platform_data mt8516_platform_data = { + .min_volt_shift = 100000, + .max_volt_shift = 200000, + .proc_max_volt = 1310000, + .sram_min_volt = 0, + .sram_max_volt = 1310000, + .ccifreq_supported = false, +}; + +/* List of machines supported by this driver */ +static const struct of_device_id mtk_cpufreq_machines[] __initconst __maybe_unused = { + { .compatible = "mediatek,mt2701", .data = &mt2701_platform_data }, + { .compatible = "mediatek,mt2712", .data = &mt2701_platform_data }, + { .compatible = "mediatek,mt7622", .data = &mt7622_platform_data }, + { .compatible = "mediatek,mt7623", .data = &mt7623_platform_data }, + { .compatible = "mediatek,mt7988a", .data = &mt7988_platform_data }, + { .compatible = "mediatek,mt8167", .data = &mt8516_platform_data }, + { .compatible = "mediatek,mt817x", .data = &mt2701_platform_data }, + { .compatible = "mediatek,mt8173", .data = &mt2701_platform_data }, + { .compatible = "mediatek,mt8176", .data = &mt2701_platform_data }, + { .compatible = "mediatek,mt8183", .data = &mt8183_platform_data }, + { .compatible = "mediatek,mt8186", .data = &mt8186_platform_data }, + { .compatible = "mediatek,mt8365", .data = &mt2701_platform_data }, + { .compatible = "mediatek,mt8516", .data = &mt8516_platform_data }, + { } +}; +MODULE_DEVICE_TABLE(of, mtk_cpufreq_machines); + +static int __init mtk_cpufreq_driver_init(void) +{ + const struct mtk_cpufreq_platform_data *data; + int err; + + data = of_machine_get_match_data(mtk_cpufreq_machines); + if (!data) { + pr_debug("Machine is not compatible with mtk-cpufreq\n"); + return -ENODEV; + } + + err = platform_driver_register(&mtk_cpufreq_platdrv); + if (err) + return err; + + /* + * Since there's no place to hold device registration code and no + * device tree based way to match cpufreq driver yet, both the driver + * and the device registration codes are put here to handle defer + * probing. + */ + cpufreq_pdev = platform_device_register_data(NULL, "mtk-cpufreq", -1, + data, sizeof(*data)); + if (IS_ERR(cpufreq_pdev)) { + pr_err("failed to register mtk-cpufreq platform device\n"); + platform_driver_unregister(&mtk_cpufreq_platdrv); + return PTR_ERR(cpufreq_pdev); + } + + return 0; +} +module_init(mtk_cpufreq_driver_init) + +static void __exit mtk_cpufreq_driver_exit(void) +{ + platform_device_unregister(cpufreq_pdev); + platform_driver_unregister(&mtk_cpufreq_platdrv); +} +module_exit(mtk_cpufreq_driver_exit) + +MODULE_DESCRIPTION("MediaTek CPUFreq driver"); +MODULE_AUTHOR("Pi-Cheng Chen <pi-cheng.chen@linaro.org>"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/cpufreq/mperf.c b/drivers/cpufreq/mperf.c deleted file mode 100644 index 911e193018ae..000000000000 --- a/drivers/cpufreq/mperf.c +++ /dev/null @@ -1,51 +0,0 @@ -#include <linux/kernel.h> -#include <linux/smp.h> -#include <linux/module.h> -#include <linux/init.h> -#include <linux/cpufreq.h> -#include <linux/slab.h> - -#include "mperf.h" - -static DEFINE_PER_CPU(struct aperfmperf, acfreq_old_perf); - -/* Called via smp_call_function_single(), on the target CPU */ -static void read_measured_perf_ctrs(void *_cur) -{ - struct aperfmperf *am = _cur; - - get_aperfmperf(am); -} - -/* - * Return the measured active (C0) frequency on this CPU since last call - * to this function. - * Input: cpu number - * Return: Average CPU frequency in terms of max frequency (zero on error) - * - * We use IA32_MPERF and IA32_APERF MSRs to get the measured performance - * over a period of time, while CPU is in C0 state. - * IA32_MPERF counts at the rate of max advertised frequency - * IA32_APERF counts at the rate of actual CPU frequency - * Only IA32_APERF/IA32_MPERF ratio is architecturally defined and - * no meaning should be associated with absolute values of these MSRs. - */ -unsigned int cpufreq_get_measured_perf(struct cpufreq_policy *policy, - unsigned int cpu) -{ - struct aperfmperf perf; - unsigned long ratio; - unsigned int retval; - - if (smp_call_function_single(cpu, read_measured_perf_ctrs, &perf, 1)) - return 0; - - ratio = calc_aperfmperf_ratio(&per_cpu(acfreq_old_perf, cpu), &perf); - per_cpu(acfreq_old_perf, cpu) = perf; - - retval = (policy->cpuinfo.max_freq * ratio) >> APERFMPERF_SHIFT; - - return retval; -} -EXPORT_SYMBOL_GPL(cpufreq_get_measured_perf); -MODULE_LICENSE("GPL"); diff --git a/drivers/cpufreq/mperf.h b/drivers/cpufreq/mperf.h deleted file mode 100644 index 5dbf2950dc22..000000000000 --- a/drivers/cpufreq/mperf.h +++ /dev/null @@ -1,9 +0,0 @@ -/* - * (c) 2010 Advanced Micro Devices, Inc. - * Your use of this code is subject to the terms and conditions of the - * GNU general public license version 2. See "COPYING" or - * http://www.gnu.org/licenses/gpl.html - */ - -unsigned int cpufreq_get_measured_perf(struct cpufreq_policy *policy, - unsigned int cpu); diff --git a/drivers/cpufreq/mvebu-cpufreq.c b/drivers/cpufreq/mvebu-cpufreq.c new file mode 100644 index 000000000000..2aad4c04673c --- /dev/null +++ b/drivers/cpufreq/mvebu-cpufreq.c @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * CPUFreq support for Armada 370/XP platforms. + * + * Copyright (C) 2012-2016 Marvell + * + * Yehuda Yitschak <yehuday@marvell.com> + * Gregory Clement <gregory.clement@free-electrons.com> + * Thomas Petazzoni <thomas.petazzoni@free-electrons.com> + */ + +#define pr_fmt(fmt) "mvebu-pmsu: " fmt + +#include <linux/clk.h> +#include <linux/cpu.h> +#include <linux/init.h> +#include <linux/kernel.h> +#include <linux/of_address.h> +#include <linux/platform_device.h> +#include <linux/pm_opp.h> +#include <linux/resource.h> + +static int __init armada_xp_pmsu_cpufreq_init(void) +{ + struct device_node *np; + struct resource res; + int ret, cpu; + + if (!of_machine_is_compatible("marvell,armadaxp")) + return 0; + + /* + * In order to have proper cpufreq handling, we need to ensure + * that the Device Tree description of the CPU clock includes + * the definition of the PMU DFS registers. If not, we do not + * register the clock notifier and the cpufreq driver. This + * piece of code is only for compatibility with old Device + * Trees. + */ + np = of_find_compatible_node(NULL, NULL, "marvell,armada-xp-cpu-clock"); + if (!np) + return 0; + + ret = of_address_to_resource(np, 1, &res); + if (ret) { + pr_warn(FW_WARN "not enabling cpufreq, deprecated armada-xp-cpu-clock binding\n"); + of_node_put(np); + return 0; + } + + of_node_put(np); + + /* + * For each CPU, this loop registers the operating points + * supported (which are the nominal CPU frequency and half of + * it), and registers the clock notifier that will take care + * of doing the PMSU part of a frequency transition. + */ + for_each_present_cpu(cpu) { + struct device *cpu_dev; + struct clk *clk; + int ret; + + cpu_dev = get_cpu_device(cpu); + if (!cpu_dev) { + pr_err("Cannot get CPU %d\n", cpu); + continue; + } + + clk = clk_get(cpu_dev, NULL); + if (IS_ERR(clk)) { + pr_err("Cannot get clock for CPU %d\n", cpu); + return PTR_ERR(clk); + } + + ret = dev_pm_opp_add(cpu_dev, clk_get_rate(clk), 0); + if (ret) { + clk_put(clk); + return ret; + } + + ret = dev_pm_opp_add(cpu_dev, clk_get_rate(clk) / 2, 0); + if (ret) { + dev_pm_opp_remove(cpu_dev, clk_get_rate(clk)); + clk_put(clk); + dev_err(cpu_dev, "Failed to register OPPs\n"); + return ret; + } + + ret = dev_pm_opp_set_sharing_cpus(cpu_dev, + cpumask_of(cpu_dev->id)); + if (ret) + dev_err(cpu_dev, "%s: failed to mark OPPs as shared: %d\n", + __func__, ret); + clk_put(clk); + } + + platform_device_register_simple("cpufreq-dt", -1, NULL, 0); + return 0; +} +device_initcall(armada_xp_pmsu_cpufreq_init); diff --git a/drivers/cpufreq/omap-cpufreq.c b/drivers/cpufreq/omap-cpufreq.c index 29468a522ee9..bbb01d93b54b 100644 --- a/drivers/cpufreq/omap-cpufreq.c +++ b/drivers/cpufreq/omap-cpufreq.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * CPU frequency scaling for OMAP using OPP information * @@ -8,11 +9,10 @@ * * Copyright (C) 2007-2011 Texas Instruments, Inc. * - OMAP3/4 support by Rajendra Nayak, Santosh Shilimkar - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + #include <linux/types.h> #include <linux/kernel.h> #include <linux/sched.h> @@ -22,80 +22,33 @@ #include <linux/err.h> #include <linux/clk.h> #include <linux/io.h> -#include <linux/opp.h> +#include <linux/pm_opp.h> #include <linux/cpu.h> #include <linux/module.h> #include <linux/platform_device.h> #include <linux/regulator/consumer.h> -#include <asm/smp_plat.h> -#include <asm/cpu.h> - /* OPP tolerance in percentage */ #define OPP_TOLERANCE 4 static struct cpufreq_frequency_table *freq_table; static atomic_t freq_table_users = ATOMIC_INIT(0); -static struct clk *mpu_clk; static struct device *mpu_dev; static struct regulator *mpu_reg; -static int omap_verify_speed(struct cpufreq_policy *policy) +static int omap_target(struct cpufreq_policy *policy, unsigned int index) { - if (!freq_table) - return -EINVAL; - return cpufreq_frequency_table_verify(policy, freq_table); -} - -static unsigned int omap_getspeed(unsigned int cpu) -{ - unsigned long rate; - - if (cpu >= NR_CPUS) - return 0; - - rate = clk_get_rate(mpu_clk) / 1000; - return rate; -} - -static int omap_target(struct cpufreq_policy *policy, - unsigned int target_freq, - unsigned int relation) -{ - unsigned int i; - int r, ret = 0; - struct cpufreq_freqs freqs; - struct opp *opp; + int r, ret; + struct dev_pm_opp *opp; unsigned long freq, volt = 0, volt_old = 0, tol = 0; + unsigned int old_freq, new_freq; - if (!freq_table) { - dev_err(mpu_dev, "%s: cpu%d: no freq table!\n", __func__, - policy->cpu); - return -EINVAL; - } - - ret = cpufreq_frequency_table_target(policy, freq_table, target_freq, - relation, &i); - if (ret) { - dev_dbg(mpu_dev, "%s: cpu%d: no freq match for %d(ret=%d)\n", - __func__, policy->cpu, target_freq, ret); - return ret; - } - freqs.new = freq_table[i].frequency; - if (!freqs.new) { - dev_err(mpu_dev, "%s: cpu%d: no match for freq %d\n", __func__, - policy->cpu, target_freq); - return -EINVAL; - } - - freqs.old = omap_getspeed(policy->cpu); - - if (freqs.old == freqs.new && policy->cur == freqs.new) - return ret; + old_freq = policy->cur; + new_freq = freq_table[index].frequency; - freq = freqs.new * 1000; - ret = clk_round_rate(mpu_clk, freq); - if (IS_ERR_VALUE(ret)) { + freq = new_freq * 1000; + ret = clk_round_rate(policy->clk, freq); + if (ret < 0) { dev_warn(mpu_dev, "CPUfreq: Cannot find matching frequency for %lu\n", freq); @@ -104,157 +57,109 @@ static int omap_target(struct cpufreq_policy *policy, freq = ret; if (mpu_reg) { - rcu_read_lock(); - opp = opp_find_freq_ceil(mpu_dev, &freq); + opp = dev_pm_opp_find_freq_ceil(mpu_dev, &freq); if (IS_ERR(opp)) { - rcu_read_unlock(); dev_err(mpu_dev, "%s: unable to find MPU OPP for %d\n", - __func__, freqs.new); + __func__, new_freq); return -EINVAL; } - volt = opp_get_voltage(opp); - rcu_read_unlock(); + volt = dev_pm_opp_get_voltage(opp); + dev_pm_opp_put(opp); tol = volt * OPP_TOLERANCE / 100; volt_old = regulator_get_voltage(mpu_reg); } dev_dbg(mpu_dev, "cpufreq-omap: %u MHz, %ld mV --> %u MHz, %ld mV\n", - freqs.old / 1000, volt_old ? volt_old / 1000 : -1, - freqs.new / 1000, volt ? volt / 1000 : -1); - - /* notifiers */ - cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE); + old_freq / 1000, volt_old ? volt_old / 1000 : -1, + new_freq / 1000, volt ? volt / 1000 : -1); /* scaling up? scale voltage before frequency */ - if (mpu_reg && (freqs.new > freqs.old)) { + if (mpu_reg && (new_freq > old_freq)) { r = regulator_set_voltage(mpu_reg, volt - tol, volt + tol); if (r < 0) { dev_warn(mpu_dev, "%s: unable to scale voltage up.\n", __func__); - freqs.new = freqs.old; - goto done; + return r; } } - ret = clk_set_rate(mpu_clk, freqs.new * 1000); + ret = clk_set_rate(policy->clk, new_freq * 1000); /* scaling down? scale voltage after frequency */ - if (mpu_reg && (freqs.new < freqs.old)) { + if (mpu_reg && (new_freq < old_freq)) { r = regulator_set_voltage(mpu_reg, volt - tol, volt + tol); if (r < 0) { dev_warn(mpu_dev, "%s: unable to scale voltage down.\n", __func__); - ret = clk_set_rate(mpu_clk, freqs.old * 1000); - freqs.new = freqs.old; - goto done; + clk_set_rate(policy->clk, old_freq * 1000); + return r; } } - freqs.new = omap_getspeed(policy->cpu); - -done: - /* notifiers */ - cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); - return ret; } static inline void freq_table_free(void) { if (atomic_dec_and_test(&freq_table_users)) - opp_free_cpufreq_table(mpu_dev, &freq_table); + dev_pm_opp_free_cpufreq_table(mpu_dev, &freq_table); } -static int __cpuinit omap_cpu_init(struct cpufreq_policy *policy) +static int omap_cpu_init(struct cpufreq_policy *policy) { - int result = 0; + int result; - mpu_clk = clk_get(NULL, "cpufreq_ck"); - if (IS_ERR(mpu_clk)) - return PTR_ERR(mpu_clk); + policy->clk = clk_get(NULL, "cpufreq_ck"); + if (IS_ERR(policy->clk)) + return PTR_ERR(policy->clk); - if (policy->cpu >= NR_CPUS) { - result = -EINVAL; - goto fail_ck; - } - - policy->cur = omap_getspeed(policy->cpu); - - if (!freq_table) - result = opp_init_cpufreq_table(mpu_dev, &freq_table); - - if (result) { - dev_err(mpu_dev, "%s: cpu%d: failed creating freq table[%d]\n", + if (!freq_table) { + result = dev_pm_opp_init_cpufreq_table(mpu_dev, &freq_table); + if (result) { + dev_err(mpu_dev, + "%s: cpu%d: failed creating freq table[%d]\n", __func__, policy->cpu, result); - goto fail_ck; + clk_put(policy->clk); + return result; + } } atomic_inc_return(&freq_table_users); - result = cpufreq_frequency_table_cpuinfo(policy, freq_table); - if (result) - goto fail_table; - - cpufreq_frequency_table_get_attr(freq_table, policy->cpu); - - policy->cur = omap_getspeed(policy->cpu); - - /* - * On OMAP SMP configuartion, both processors share the voltage - * and clock. So both CPUs needs to be scaled together and hence - * needs software co-ordination. Use cpufreq affected_cpus - * interface to handle this scenario. Additional is_smp() check - * is to keep SMP_ON_UP build working. - */ - if (is_smp()) - cpumask_setall(policy->cpus); - /* FIXME: what's the actual transition time? */ - policy->cpuinfo.transition_latency = 300 * 1000; + cpufreq_generic_init(policy, freq_table, 300 * 1000); return 0; - -fail_table: - freq_table_free(); -fail_ck: - clk_put(mpu_clk); - return result; } -static int omap_cpu_exit(struct cpufreq_policy *policy) +static void omap_cpu_exit(struct cpufreq_policy *policy) { freq_table_free(); - clk_put(mpu_clk); - return 0; + clk_put(policy->clk); } -static struct freq_attr *omap_cpufreq_attr[] = { - &cpufreq_freq_attr_scaling_available_freqs, - NULL, -}; - static struct cpufreq_driver omap_driver = { - .flags = CPUFREQ_STICKY, - .verify = omap_verify_speed, - .target = omap_target, - .get = omap_getspeed, + .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK, + .verify = cpufreq_generic_frequency_table_verify, + .target_index = omap_target, + .get = cpufreq_generic_get, .init = omap_cpu_init, .exit = omap_cpu_exit, + .register_em = cpufreq_register_em_with_opp, .name = "omap", - .attr = omap_cpufreq_attr, }; static int omap_cpufreq_probe(struct platform_device *pdev) { mpu_dev = get_cpu_device(0); if (!mpu_dev) { - pr_warning("%s: unable to get the mpu device\n", __func__); + pr_warn("%s: unable to get the MPU device\n", __func__); return -EINVAL; } mpu_reg = regulator_get(mpu_dev, "vcc"); if (IS_ERR(mpu_reg)) { - pr_warning("%s: unable to get MPU regulator\n", __func__); + pr_warn("%s: unable to get MPU regulator\n", __func__); mpu_reg = NULL; } else { /* @@ -272,15 +177,14 @@ static int omap_cpufreq_probe(struct platform_device *pdev) return cpufreq_register_driver(&omap_driver); } -static int omap_cpufreq_remove(struct platform_device *pdev) +static void omap_cpufreq_remove(struct platform_device *pdev) { - return cpufreq_unregister_driver(&omap_driver); + cpufreq_unregister_driver(&omap_driver); } static struct platform_driver omap_cpufreq_platdrv = { .driver = { .name = "omap-cpufreq", - .owner = THIS_MODULE, }, .probe = omap_cpufreq_probe, .remove = omap_cpufreq_remove, diff --git a/drivers/cpufreq/p4-clockmod.c b/drivers/cpufreq/p4-clockmod.c index 9ee78170ff86..69c19233fcd4 100644 --- a/drivers/cpufreq/p4-clockmod.c +++ b/drivers/cpufreq/p4-clockmod.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-or-later /* * Pentium 4/Xeon CPU on demand clock modulation/speed scaling * (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de> @@ -6,20 +7,16 @@ * (C) 2002 Tora T. Engstad * All Rights Reserved * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - * * The author(s) of this software shall not be held liable for damages * of any nature resulting due to the use of this software. This * software is provided AS-IS with no warranties. * * Date Errata Description * 20020525 N44, O17 12.5% or 25% DC causes lockup - * */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + #include <linux/kernel.h> #include <linux/module.h> #include <linux/init.h> @@ -35,8 +32,6 @@ #include "speedstep-lib.h" -#define PFX "p4-clockmod: " - /* * Duty Cycle (3bits), note DC_DISABLE is not specified in * intel docs i just use it to mean disable @@ -92,69 +87,39 @@ static int cpufreq_p4_setdc(unsigned int cpu, unsigned int newstate) static struct cpufreq_frequency_table p4clockmod_table[] = { - {DC_RESV, CPUFREQ_ENTRY_INVALID}, - {DC_DFLT, 0}, - {DC_25PT, 0}, - {DC_38PT, 0}, - {DC_50PT, 0}, - {DC_64PT, 0}, - {DC_75PT, 0}, - {DC_88PT, 0}, - {DC_DISABLE, 0}, - {DC_RESV, CPUFREQ_TABLE_END}, + {0, DC_RESV, CPUFREQ_ENTRY_INVALID}, + {0, DC_DFLT, 0}, + {0, DC_25PT, 0}, + {0, DC_38PT, 0}, + {0, DC_50PT, 0}, + {0, DC_64PT, 0}, + {0, DC_75PT, 0}, + {0, DC_88PT, 0}, + {0, DC_DISABLE, 0}, + {0, DC_RESV, CPUFREQ_TABLE_END}, }; -static int cpufreq_p4_target(struct cpufreq_policy *policy, - unsigned int target_freq, - unsigned int relation) +static int cpufreq_p4_target(struct cpufreq_policy *policy, unsigned int index) { - unsigned int newstate = DC_RESV; - struct cpufreq_freqs freqs; int i; - if (cpufreq_frequency_table_target(policy, &p4clockmod_table[0], - target_freq, relation, &newstate)) - return -EINVAL; - - freqs.old = cpufreq_p4_get(policy->cpu); - freqs.new = stock_freq * p4clockmod_table[newstate].driver_data / 8; - - if (freqs.new == freqs.old) - return 0; - - /* notifiers */ - cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE); - /* run on each logical CPU, * see section 13.15.3 of IA32 Intel Architecture Software * Developer's Manual, Volume 3 */ for_each_cpu(i, policy->cpus) - cpufreq_p4_setdc(i, p4clockmod_table[newstate].driver_data); - - /* notifiers */ - cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); + cpufreq_p4_setdc(i, p4clockmod_table[index].driver_data); return 0; } -static int cpufreq_p4_verify(struct cpufreq_policy *policy) -{ - return cpufreq_frequency_table_verify(policy, &p4clockmod_table[0]); -} - - static unsigned int cpufreq_p4_get_frequency(struct cpuinfo_x86 *c) { if (c->x86 == 0x06) { if (cpu_has(c, X86_FEATURE_EST)) - printk_once(KERN_WARNING PFX "Warning: EST-capable " - "CPU detected. The acpi-cpufreq module offers " - "voltage scaling in addition to frequency " - "scaling. You should use that instead of " - "p4-clockmod, if possible.\n"); + pr_warn_once("Warning: EST-capable CPU detected. The acpi-cpufreq module offers voltage scaling in addition to frequency scaling. You should use that instead of p4-clockmod, if possible.\n"); switch (c->x86_model) { case 0x0E: /* Core */ case 0x0F: /* Core Duo */ @@ -164,7 +129,7 @@ static unsigned int cpufreq_p4_get_frequency(struct cpuinfo_x86 *c) return speedstep_get_frequency(SPEEDSTEP_CPU_PCORE); case 0x0D: /* Pentium M (Dothan) */ p4clockmod_driver.flags |= CPUFREQ_CONST_LOOPS; - /* fall through */ + fallthrough; case 0x09: /* Pentium M (Banias) */ return speedstep_get_frequency(SPEEDSTEP_CPU_PM); } @@ -178,11 +143,7 @@ static unsigned int cpufreq_p4_get_frequency(struct cpuinfo_x86 *c) p4clockmod_driver.flags |= CPUFREQ_CONST_LOOPS; if (speedstep_detect_processor() == SPEEDSTEP_CPU_P4M) { - printk(KERN_WARNING PFX "Warning: Pentium 4-M detected. " - "The speedstep-ich or acpi cpufreq modules offer " - "voltage scaling in addition of frequency scaling. " - "You should use either one instead of p4-clockmod, " - "if possible.\n"); + pr_warn("Warning: Pentium 4-M detected. The speedstep-ich or acpi cpufreq modules offer voltage scaling in addition of frequency scaling. You should use either one instead of p4-clockmod, if possible.\n"); return speedstep_get_frequency(SPEEDSTEP_CPU_P4M); } @@ -198,11 +159,11 @@ static int cpufreq_p4_cpu_init(struct cpufreq_policy *policy) unsigned int i; #ifdef CONFIG_SMP - cpumask_copy(policy->cpus, cpu_sibling_mask(policy->cpu)); + cpumask_copy(policy->cpus, topology_sibling_cpumask(policy->cpu)); #endif /* Errata workaround */ - cpuid = (c->x86 << 8) | (c->x86_model << 4) | c->x86_mask; + cpuid = (c->x86 << 8) | (c->x86_model << 4) | c->x86_stepping; switch (cpuid) { case 0x0f07: case 0x0f0a: @@ -230,25 +191,18 @@ static int cpufreq_p4_cpu_init(struct cpufreq_policy *policy) else p4clockmod_table[i].frequency = (stock_freq * i)/8; } - cpufreq_frequency_table_get_attr(p4clockmod_table, policy->cpu); /* cpuinfo and default policy values */ /* the transition latency is set to be 1 higher than the maximum * transition latency of the ondemand governor */ policy->cpuinfo.transition_latency = 10000001; - policy->cur = stock_freq; - - return cpufreq_frequency_table_cpuinfo(policy, &p4clockmod_table[0]); -} + policy->freq_table = &p4clockmod_table[0]; - -static int cpufreq_p4_cpu_exit(struct cpufreq_policy *policy) -{ - cpufreq_frequency_table_put_attr(policy->cpu); return 0; } + static unsigned int cpufreq_p4_get(unsigned int cpu) { u32 l, h; @@ -267,24 +221,16 @@ static unsigned int cpufreq_p4_get(unsigned int cpu) return stock_freq; } -static struct freq_attr *p4clockmod_attr[] = { - &cpufreq_freq_attr_scaling_available_freqs, - NULL, -}; - static struct cpufreq_driver p4clockmod_driver = { - .verify = cpufreq_p4_verify, - .target = cpufreq_p4_target, + .verify = cpufreq_generic_frequency_table_verify, + .target_index = cpufreq_p4_target, .init = cpufreq_p4_cpu_init, - .exit = cpufreq_p4_cpu_exit, .get = cpufreq_p4_get, .name = "p4-clockmod", - .owner = THIS_MODULE, - .attr = p4clockmod_attr, }; static const struct x86_cpu_id cpufreq_p4_id[] = { - { X86_VENDOR_INTEL, X86_FAMILY_ANY, X86_MODEL_ANY, X86_FEATURE_ACC }, + X86_MATCH_VENDOR_FEATURE(INTEL, X86_FEATURE_ACC, NULL), {} }; @@ -306,8 +252,7 @@ static int __init cpufreq_p4_init(void) ret = cpufreq_register_driver(&p4clockmod_driver); if (!ret) - printk(KERN_INFO PFX "P4/Xeon(TM) CPU On-Demand Clock " - "Modulation available\n"); + pr_info("P4/Xeon(TM) CPU On-Demand Clock Modulation available\n"); return ret; } diff --git a/drivers/cpufreq/pasemi-cpufreq.c b/drivers/cpufreq/pasemi-cpufreq.c index b704da404067..a3931349360f 100644 --- a/drivers/cpufreq/pasemi-cpufreq.c +++ b/drivers/cpufreq/pasemi-cpufreq.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-or-later /* * Copyright (C) 2007 PA Semi, Inc * @@ -8,33 +9,20 @@ * * Based on arch/powerpc/platforms/cell/cbe_cpufreq.c: * (C) Copyright IBM Deutschland Entwicklung GmbH 2005 - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2, or (at your option) - * any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - * */ #include <linux/cpufreq.h> #include <linux/timer.h> #include <linux/module.h> +#include <linux/of_address.h> #include <asm/hw_irq.h> #include <asm/io.h> -#include <asm/prom.h> #include <asm/time.h> #include <asm/smp.h> +#include <platforms/pasemi/pasemi.h> + #define SDCASR_REG 0x0100 #define SDCASR_REG_STRIDE 0x1000 #define SDCPWR_CFGA0_REG 0x0100 @@ -51,8 +39,6 @@ static void __iomem *sdcpwr_mapbase; static void __iomem *sdcasr_mapbase; -static DEFINE_MUTEX(pas_switch_mutex); - /* Current astate, is used when waking up from power savings on * one core, in case the other core has switched states during * the idle time. @@ -61,17 +47,12 @@ static int current_astate; /* We support 5(A0-A4) power states excluding turbo(A5-A6) modes */ static struct cpufreq_frequency_table pas_freqs[] = { - {0, 0}, - {1, 0}, - {2, 0}, - {3, 0}, - {4, 0}, - {0, CPUFREQ_TABLE_END}, -}; - -static struct freq_attr *pas_cpu_freqs_attr[] = { - &cpufreq_freq_attr_scaling_available_freqs, - NULL, + {0, 0, 0}, + {0, 1, 0}, + {0, 2, 0}, + {0, 3, 0}, + {0, 4, 0}, + {0, 0, CPUFREQ_TABLE_END}, }; /* @@ -142,18 +123,28 @@ void restore_astate(int cpu) static int pas_cpufreq_cpu_init(struct cpufreq_policy *policy) { + struct cpufreq_frequency_table *pos; const u32 *max_freqp; u32 max_freq; - int i, cur_astate; + int cur_astate, idx; struct resource res; struct device_node *cpu, *dn; int err = -ENODEV; cpu = of_get_cpu_node(policy->cpu, NULL); - if (!cpu) goto out; + max_freqp = of_get_property(cpu, "clock-frequency", NULL); + of_node_put(cpu); + if (!max_freqp) { + err = -EINVAL; + goto out; + } + + /* we need the freq in kHz */ + max_freq = *max_freqp / 1000; + dn = of_find_compatible_node(NULL, NULL, "1682m-sdc"); if (!dn) dn = of_find_compatible_node(NULL, NULL, @@ -189,45 +180,23 @@ static int pas_cpufreq_cpu_init(struct cpufreq_policy *policy) } pr_debug("init cpufreq on CPU %d\n", policy->cpu); - - max_freqp = of_get_property(cpu, "clock-frequency", NULL); - if (!max_freqp) { - err = -EINVAL; - goto out_unmap_sdcpwr; - } - - /* we need the freq in kHz */ - max_freq = *max_freqp / 1000; - pr_debug("max clock-frequency is at %u kHz\n", max_freq); pr_debug("initializing frequency table\n"); /* initialize frequency table */ - for (i=0; pas_freqs[i].frequency!=CPUFREQ_TABLE_END; i++) { - pas_freqs[i].frequency = - get_astate_freq(pas_freqs[i].driver_data) * 100000; - pr_debug("%d: %d\n", i, pas_freqs[i].frequency); + cpufreq_for_each_entry_idx(pos, pas_freqs, idx) { + pos->frequency = get_astate_freq(pos->driver_data) * 100000; + pr_debug("%d: %d\n", idx, pos->frequency); } - policy->cpuinfo.transition_latency = get_gizmo_latency(); - cur_astate = get_cur_astate(policy->cpu); pr_debug("current astate is at %d\n",cur_astate); policy->cur = pas_freqs[cur_astate].frequency; - cpumask_copy(policy->cpus, cpu_online_mask); - ppc_proc_freq = policy->cur * 1000ul; - cpufreq_frequency_table_get_attr(pas_freqs, policy->cpu); - - /* this ensures that policy->cpuinfo_min and policy->cpuinfo_max - * are set correctly - */ - return cpufreq_frequency_table_cpuinfo(policy, pas_freqs); - -out_unmap_sdcpwr: - iounmap(sdcpwr_mapbase); + cpufreq_generic_init(policy, pas_freqs, get_gizmo_latency()); + return 0; out_unmap_sdcasr: iounmap(sdcasr_mapbase); @@ -235,49 +204,26 @@ out: return err; } -static int pas_cpufreq_cpu_exit(struct cpufreq_policy *policy) +static void pas_cpufreq_cpu_exit(struct cpufreq_policy *policy) { /* * We don't support CPU hotplug. Don't unmap after the system * has already made it to a running state. */ - if (system_state != SYSTEM_BOOTING) - return 0; + if (system_state >= SYSTEM_RUNNING) + return; if (sdcasr_mapbase) iounmap(sdcasr_mapbase); if (sdcpwr_mapbase) iounmap(sdcpwr_mapbase); - - cpufreq_frequency_table_put_attr(policy->cpu); - return 0; -} - -static int pas_cpufreq_verify(struct cpufreq_policy *policy) -{ - return cpufreq_frequency_table_verify(policy, pas_freqs); } static int pas_cpufreq_target(struct cpufreq_policy *policy, - unsigned int target_freq, - unsigned int relation) + unsigned int pas_astate_new) { - struct cpufreq_freqs freqs; - int pas_astate_new; int i; - cpufreq_frequency_table_target(policy, - pas_freqs, - target_freq, - relation, - &pas_astate_new); - - freqs.old = policy->cur; - freqs.new = pas_freqs[pas_astate_new].frequency; - - mutex_lock(&pas_switch_mutex); - cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE); - pr_debug("setting frequency for cpu %d to %d kHz, 1/%d of max frequency\n", policy->cpu, pas_freqs[pas_astate_new].frequency, @@ -288,22 +234,17 @@ static int pas_cpufreq_target(struct cpufreq_policy *policy, for_each_online_cpu(i) set_astate(i, pas_astate_new); - cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); - mutex_unlock(&pas_switch_mutex); - - ppc_proc_freq = freqs.new * 1000ul; + ppc_proc_freq = pas_freqs[pas_astate_new].frequency * 1000ul; return 0; } static struct cpufreq_driver pas_cpufreq_driver = { .name = "pas-cpufreq", - .owner = THIS_MODULE, .flags = CPUFREQ_CONST_LOOPS, .init = pas_cpufreq_cpu_init, .exit = pas_cpufreq_cpu_exit, - .verify = pas_cpufreq_verify, - .target = pas_cpufreq_target, - .attr = pas_cpu_freqs_attr, + .verify = cpufreq_generic_frequency_table_verify, + .target_index = pas_cpufreq_target, }; /* @@ -327,5 +268,6 @@ static void __exit pas_cpufreq_exit(void) module_init(pas_cpufreq_init); module_exit(pas_cpufreq_exit); +MODULE_DESCRIPTION("cpufreq driver for PA Semi PWRficient"); MODULE_LICENSE("GPL"); MODULE_AUTHOR("Egor Martovetsky <egor@pasemi.com>, Olof Johansson <olof@lixom.net>"); diff --git a/drivers/cpufreq/pcc-cpufreq.c b/drivers/cpufreq/pcc-cpufreq.c index 1581fcc4cf4a..ac2e90a65f0c 100644 --- a/drivers/cpufreq/pcc-cpufreq.c +++ b/drivers/cpufreq/pcc-cpufreq.c @@ -31,6 +31,7 @@ #include <linux/cpufreq.h> #include <linux/compiler.h> #include <linux/slab.h> +#include <linux/platform_device.h> #include <linux/acpi.h> #include <linux/io.h> @@ -109,10 +110,9 @@ struct pcc_cpu { static struct pcc_cpu __percpu *pcc_cpu_info; -static int pcc_cpufreq_verify(struct cpufreq_policy *policy) +static int pcc_cpufreq_verify(struct cpufreq_policy_data *policy) { - cpufreq_verify_within_limits(policy, policy->cpuinfo.min_freq, - policy->cpuinfo.max_freq); + cpufreq_verify_within_cpu_limits(policy); return 0; } @@ -205,7 +205,6 @@ static int pcc_cpufreq_target(struct cpufreq_policy *policy, u32 input_buffer; int cpu; - spin_lock(&pcc_lock); cpu = policy->cpu; pcc_cpu_data = per_cpu_ptr(pcc_cpu_info, cpu); @@ -214,8 +213,10 @@ static int pcc_cpufreq_target(struct cpufreq_policy *policy, cpu, target_freq, (pcch_virt_addr + pcc_cpu_data->input_offset)); + freqs.old = policy->cur; freqs.new = target_freq; - cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE); + cpufreq_freq_transition_begin(policy, &freqs); + spin_lock(&pcc_lock); input_buffer = 0x1 | (((target_freq * 100) / (ioread32(&pcch_hdr->nominal) * 1000)) << 8); @@ -229,25 +230,20 @@ static int pcc_cpufreq_target(struct cpufreq_policy *policy, memset_io((pcch_virt_addr + pcc_cpu_data->input_offset), 0, BUF_SZ); status = ioread16(&pcch_hdr->status); + iowrite16(0, &pcch_hdr->status); + + spin_unlock(&pcc_lock); + cpufreq_freq_transition_end(policy, &freqs, status != CMD_COMPLETE); + if (status != CMD_COMPLETE) { pr_debug("target: FAILED for cpu %d, with status: 0x%x\n", cpu, status); - goto cmd_incomplete; + return -EINVAL; } - iowrite16(0, &pcch_hdr->status); - cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); pr_debug("target: was SUCCESSFUL for cpu %d\n", cpu); - spin_unlock(&pcc_lock); return 0; - -cmd_incomplete: - freqs.new = freqs.old; - cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); - iowrite16(0, &pcch_hdr->status); - spin_unlock(&pcc_lock); - return -EINVAL; } static int pcc_get_offset(int cpu) @@ -273,7 +269,7 @@ static int pcc_get_offset(int cpu) if (!pccp || pccp->type != ACPI_TYPE_PACKAGE) { ret = -ENODEV; goto out_free; - }; + } offset = &(pccp->package.elements[0]); if (!offset || offset->type != ACPI_TYPE_INTEGER) { @@ -389,22 +385,21 @@ out_free: return ret; } -static int __init pcc_cpufreq_probe(void) +static int __init pcc_cpufreq_evaluate(void) { acpi_status status; struct acpi_buffer output = {ACPI_ALLOCATE_BUFFER, NULL}; struct pcc_memory_resource *mem_resource; struct pcc_register_resource *reg_resource; union acpi_object *out_obj, *member; - acpi_handle handle, osc_handle, pcch_handle; + acpi_handle handle, osc_handle; int ret = 0; status = acpi_get_handle(NULL, "\\_SB", &handle); if (ACPI_FAILURE(status)) return -ENODEV; - status = acpi_get_handle(handle, "PCCH", &pcch_handle); - if (ACPI_FAILURE(status)) + if (!acpi_has_method(handle, "PCCH")) return -ENODEV; status = acpi_get_handle(handle, "_OSC", &osc_handle); @@ -451,7 +446,7 @@ static int __init pcc_cpufreq_probe(void) goto out_free; } - pcch_virt_addr = ioremap_nocache(mem_resource->minimum, + pcch_virt_addr = ioremap(mem_resource->minimum, mem_resource->address_length); if (pcch_virt_addr == NULL) { pr_debug("probe: could not map shared mem region\n"); @@ -493,7 +488,7 @@ static int __init pcc_cpufreq_probe(void) doorbell.space_id = reg_resource->space_id; doorbell.bit_width = reg_resource->bit_width; doorbell.bit_offset = reg_resource->bit_offset; - doorbell.access_width = 64; + doorbell.access_width = 4; doorbell.address = reg_resource->address; pr_debug("probe: doorbell: space_id is %d, bit_width is %d, " @@ -560,13 +555,6 @@ static int pcc_cpufreq_cpu_init(struct cpufreq_policy *policy) ioread32(&pcch_hdr->nominal) * 1000; policy->min = policy->cpuinfo.min_freq = ioread32(&pcch_hdr->minimum_frequency) * 1000; - policy->cur = pcc_get_freq(cpu); - - if (!policy->cur) { - pr_debug("init: Unable to get current CPU frequency\n"); - result = -EINVAL; - goto out; - } pr_debug("init: policy->max is %d, policy->min is %d\n", policy->max, policy->min); @@ -574,41 +562,47 @@ out: return result; } -static int pcc_cpufreq_cpu_exit(struct cpufreq_policy *policy) -{ - return 0; -} - static struct cpufreq_driver pcc_cpufreq_driver = { .flags = CPUFREQ_CONST_LOOPS, .get = pcc_get_freq, .verify = pcc_cpufreq_verify, .target = pcc_cpufreq_target, .init = pcc_cpufreq_cpu_init, - .exit = pcc_cpufreq_cpu_exit, .name = "pcc-cpufreq", - .owner = THIS_MODULE, }; -static int __init pcc_cpufreq_init(void) +static int __init pcc_cpufreq_probe(struct platform_device *pdev) { int ret; + /* Skip initialization if another cpufreq driver is there. */ + if (cpufreq_get_current_driver()) + return -ENODEV; + if (acpi_disabled) - return 0; + return -ENODEV; - ret = pcc_cpufreq_probe(); + ret = pcc_cpufreq_evaluate(); if (ret) { - pr_debug("pcc_cpufreq_init: PCCH evaluation failed\n"); + pr_debug("pcc_cpufreq_probe: PCCH evaluation failed\n"); return ret; } + if (num_present_cpus() > 4) { + pcc_cpufreq_driver.flags |= CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING; + pr_err("%s: Too many CPUs, dynamic performance scaling disabled\n", + __func__); + pr_err("%s: Try to enable another scaling driver through BIOS settings\n", + __func__); + pr_err("%s: and complain to the system vendor\n", __func__); + } + ret = cpufreq_register_driver(&pcc_cpufreq_driver); return ret; } -static void __exit pcc_cpufreq_exit(void) +static void pcc_cpufreq_remove(struct platform_device *pdev) { cpufreq_unregister_driver(&pcc_cpufreq_driver); @@ -617,6 +611,25 @@ static void __exit pcc_cpufreq_exit(void) free_percpu(pcc_cpu_info); } +static struct platform_driver pcc_cpufreq_platdrv = { + .driver = { + .name = "pcc-cpufreq", + }, + .remove = pcc_cpufreq_remove, +}; + +static int __init pcc_cpufreq_init(void) +{ + return platform_driver_probe(&pcc_cpufreq_platdrv, pcc_cpufreq_probe); +} + +static void __exit pcc_cpufreq_exit(void) +{ + platform_driver_unregister(&pcc_cpufreq_platdrv); +} + +MODULE_ALIAS("platform:pcc-cpufreq"); + MODULE_AUTHOR("Matthew Garrett, Naga Chumbalkar"); MODULE_VERSION(PCC_VERSION); MODULE_DESCRIPTION("Processor Clocking Control interface driver"); diff --git a/drivers/cpufreq/pmac32-cpufreq.c b/drivers/cpufreq/pmac32-cpufreq.c index 3104fad82480..a22c22bd693a 100644 --- a/drivers/cpufreq/pmac32-cpufreq.c +++ b/drivers/cpufreq/pmac32-cpufreq.c @@ -1,18 +1,16 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2002 - 2005 Benjamin Herrenschmidt <benh@kernel.crashing.org> * Copyright (C) 2004 John Steele Scott <toojays@toojays.net> * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * * TODO: Need a big cleanup here. Basically, we need to have different * cpufreq_driver structures for the different type of HW instead of the * current mess. We also need to better deal with the detection of the * type of machine. - * */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + #include <linux/module.h> #include <linux/types.h> #include <linux/errno.h> @@ -25,7 +23,9 @@ #include <linux/init.h> #include <linux/device.h> #include <linux/hardirq.h> -#include <asm/prom.h> +#include <linux/of.h> +#include <linux/of_address.h> + #include <asm/machdep.h> #include <asm/irq.h> #include <asm/pmac_feature.h> @@ -80,14 +80,9 @@ static int is_pmu_based; #define CPUFREQ_LOW 1 static struct cpufreq_frequency_table pmac_cpu_freqs[] = { - {CPUFREQ_HIGH, 0}, - {CPUFREQ_LOW, 0}, - {0, CPUFREQ_TABLE_END}, -}; - -static struct freq_attr* pmac_cpu_freqs_attr[] = { - &cpufreq_freq_attr_scaling_available_freqs, - NULL, + {0, CPUFREQ_HIGH, 0}, + {0, CPUFREQ_LOW, 0}, + {0, 0, CPUFREQ_TABLE_END}, }; static inline void local_delay(unsigned long ms) @@ -125,20 +120,20 @@ static int cpu_750fx_cpu_speed(int low_speed) /* tweak L2 for high voltage */ if (has_cpu_l2lve) { - hid2 = mfspr(SPRN_HID2); + hid2 = mfspr(SPRN_HID2_750FX); hid2 &= ~0x2000; - mtspr(SPRN_HID2, hid2); + mtspr(SPRN_HID2_750FX, hid2); } } -#ifdef CONFIG_6xx +#ifdef CONFIG_PPC_BOOK3S_32 low_choose_750fx_pll(low_speed); #endif if (low_speed == 1) { /* tweak L2 for low voltage */ if (has_cpu_l2lve) { - hid2 = mfspr(SPRN_HID2); + hid2 = mfspr(SPRN_HID2_750FX); hid2 |= 0x2000; - mtspr(SPRN_HID2, hid2); + mtspr(SPRN_HID2_750FX, hid2); } /* ramping down, set voltage last */ @@ -168,7 +163,7 @@ static int dfs_set_cpu_speed(int low_speed) } /* set frequency */ -#ifdef CONFIG_6xx +#ifdef CONFIG_PPC_BOOK3S_32 low_choose_7447a_dfs(low_speed); #endif udelay(100); @@ -302,7 +297,7 @@ static int pmu_set_cpu_speed(int low_speed) _set_L3CR(save_l3cr); /* Restore userland MMU context */ - switch_mmu_context(NULL, current->active_mm); + switch_mmu_context(NULL, current->active_mm, NULL); #ifdef DEBUG_FREQ printk(KERN_DEBUG "HID1, after: %x\n", mfspr(SPRN_HID1)); @@ -335,21 +330,11 @@ static int pmu_set_cpu_speed(int low_speed) return 0; } -static int do_set_cpu_speed(struct cpufreq_policy *policy, int speed_mode, - int notify) +static int do_set_cpu_speed(struct cpufreq_policy *policy, int speed_mode) { - struct cpufreq_freqs freqs; unsigned long l3cr; static unsigned long prev_l3cr; - freqs.old = cur_freq; - freqs.new = (speed_mode == CPUFREQ_HIGH) ? hi_freq : low_freq; - - if (freqs.old == freqs.new) - return 0; - - if (notify) - cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE); if (speed_mode == CPUFREQ_LOW && cpu_has_feature(CPU_FTR_L3CR)) { l3cr = _get_L3CR(); @@ -365,8 +350,6 @@ static int do_set_cpu_speed(struct cpufreq_policy *policy, int speed_mode, if ((prev_l3cr & L3CR_L3E) && l3cr != prev_l3cr) _set_L3CR(prev_l3cr); } - if (notify) - cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); cur_freq = (speed_mode == CPUFREQ_HIGH) ? hi_freq : low_freq; return 0; @@ -377,23 +360,12 @@ static unsigned int pmac_cpufreq_get_speed(unsigned int cpu) return cur_freq; } -static int pmac_cpufreq_verify(struct cpufreq_policy *policy) -{ - return cpufreq_frequency_table_verify(policy, pmac_cpu_freqs); -} - static int pmac_cpufreq_target( struct cpufreq_policy *policy, - unsigned int target_freq, - unsigned int relation) + unsigned int index) { - unsigned int newstate = 0; int rc; - if (cpufreq_frequency_table_target(policy, pmac_cpu_freqs, - target_freq, relation, &newstate)) - return -EINVAL; - - rc = do_set_cpu_speed(policy, newstate, 1); + rc = do_set_cpu_speed(policy, index); ppc_proc_freq = cur_freq * 1000ul; return rc; @@ -401,22 +373,15 @@ static int pmac_cpufreq_target( struct cpufreq_policy *policy, static int pmac_cpufreq_cpu_init(struct cpufreq_policy *policy) { - if (policy->cpu != 0) - return -ENODEV; - - policy->cpuinfo.transition_latency = transition_latency; - policy->cur = cur_freq; - - cpufreq_frequency_table_get_attr(pmac_cpu_freqs, policy->cpu); - return cpufreq_frequency_table_cpuinfo(policy, pmac_cpu_freqs); + cpufreq_generic_init(policy, pmac_cpu_freqs, transition_latency); + return 0; } static u32 read_gpio(struct device_node *np) { - const u32 *reg = of_get_property(np, "reg", NULL); - u32 offset; + u64 offset; - if (reg == NULL) + if (of_property_read_reg(np, 0, &offset, NULL) < 0) return 0; /* That works for all keylargos but shall be fixed properly * some day... The problem is that it seems we can't rely @@ -424,7 +389,6 @@ static u32 read_gpio(struct device_node *np) * relative to the base of KeyLargo or to the base of the * GPIO space, and the device-tree doesn't help. */ - offset = *reg; if (offset < KEYLARGO_GPIO_LEVELS0) offset += KEYLARGO_GPIO_LEVELS0; return offset; @@ -442,7 +406,7 @@ static int pmac_cpufreq_suspend(struct cpufreq_policy *policy) no_schedule = 1; sleep_freq = cur_freq; if (cur_freq == low_freq && !is_pmu_based) - do_set_cpu_speed(policy, CPUFREQ_HIGH, 0); + do_set_cpu_speed(policy, CPUFREQ_HIGH); return 0; } @@ -459,7 +423,7 @@ static int pmac_cpufreq_resume(struct cpufreq_policy *policy) * probably high speed due to our suspend() routine */ do_set_cpu_speed(policy, sleep_freq == low_freq ? - CPUFREQ_LOW : CPUFREQ_HIGH, 0); + CPUFREQ_LOW : CPUFREQ_HIGH); ppc_proc_freq = cur_freq * 1000ul; @@ -468,16 +432,14 @@ static int pmac_cpufreq_resume(struct cpufreq_policy *policy) } static struct cpufreq_driver pmac_cpufreq_driver = { - .verify = pmac_cpufreq_verify, - .target = pmac_cpufreq_target, + .verify = cpufreq_generic_frequency_table_verify, + .target_index = pmac_cpufreq_target, .get = pmac_cpufreq_get_speed, .init = pmac_cpufreq_cpu_init, .suspend = pmac_cpufreq_suspend, .resume = pmac_cpufreq_resume, - .flags = CPUFREQ_PM_NO_WARN, - .attr = pmac_cpu_freqs_attr, + .flags = CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING, .name = "powermac", - .owner = THIS_MODULE, }; @@ -506,6 +468,10 @@ static int pmac_cpufreq_init_MacRISC3(struct device_node *cpunode) if (slew_done_gpio_np) slew_done_gpio = read_gpio(slew_done_gpio_np); + of_node_put(volt_gpio_np); + of_node_put(freq_gpio_np); + of_node_put(slew_done_gpio_np); + /* If we use the frequency GPIOs, calculate the min/max speeds based * on the bus frequencies */ @@ -516,13 +482,13 @@ static int pmac_cpufreq_init_MacRISC3(struct device_node *cpunode) freqs = of_get_property(cpunode, "bus-frequencies", &lenp); lenp /= sizeof(u32); if (freqs == NULL || lenp != 2) { - printk(KERN_ERR "cpufreq: bus-frequencies incorrect or missing\n"); + pr_err("bus-frequencies incorrect or missing\n"); return 1; } ratio = of_get_property(cpunode, "processor-to-bus-ratio*2", NULL); if (ratio == NULL) { - printk(KERN_ERR "cpufreq: processor-to-bus-ratio*2 missing\n"); + pr_err("processor-to-bus-ratio*2 missing\n"); return 1; } @@ -578,14 +544,15 @@ static int pmac_cpufreq_init_7447A(struct device_node *cpunode) { struct device_node *volt_gpio_np; - if (of_get_property(cpunode, "dynamic-power-step", NULL) == NULL) + if (!of_property_read_bool(cpunode, "dynamic-power-step")) return 1; volt_gpio_np = of_find_node_by_name(NULL, "cpu-vcore-select"); if (volt_gpio_np) voltage_gpio = read_gpio(volt_gpio_np); + of_node_put(volt_gpio_np); if (!voltage_gpio){ - printk(KERN_ERR "cpufreq: missing cpu-vcore-select gpio\n"); + pr_err("missing cpu-vcore-select gpio\n"); return 1; } @@ -607,7 +574,7 @@ static int pmac_cpufreq_init_750FX(struct device_node *cpunode) u32 pvr; const u32 *value; - if (of_get_property(cpunode, "dynamic-power-step", NULL) == NULL) + if (!of_property_read_bool(cpunode, "dynamic-power-step")) return 1; hi_freq = cur_freq; @@ -620,6 +587,7 @@ static int pmac_cpufreq_init_750FX(struct device_node *cpunode) if (volt_gpio_np) voltage_gpio = read_gpio(volt_gpio_np); + of_node_put(volt_gpio_np); pvr = mfspr(SPRN_PVR); has_cpu_l2lve = !((pvr & 0xf00) == 0x100); @@ -646,11 +614,11 @@ static int __init pmac_cpufreq_setup(void) struct device_node *cpunode; const u32 *value; - if (strstr(cmd_line, "nocpufreq")) + if (strstr(boot_command_line, "nocpufreq")) return 0; - /* Assume only one CPU */ - cpunode = of_find_node_by_type(NULL, "cpu"); + /* Get first CPU node */ + cpunode = of_cpu_device_node_get(0); if (!cpunode) goto out; @@ -659,14 +627,16 @@ static int __init pmac_cpufreq_setup(void) if (!value) goto out; cur_freq = (*value) / 1000; - transition_latency = CPUFREQ_ETERNAL; /* Check for 7447A based MacRISC3 */ if (of_machine_is_compatible("MacRISC3") && - of_get_property(cpunode, "dynamic-power-step", NULL) && + of_property_read_bool(cpunode, "dynamic-power-step") && PVR_VER(mfspr(SPRN_PVR)) == 0x8003) { pmac_cpufreq_init_7447A(cpunode); + + /* Allow dynamic switching */ transition_latency = 8000000; + pmac_cpufreq_driver.flags &= ~CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING; /* Check for other MacRISC3 machines */ } else if (of_machine_is_compatible("PowerBook3,4") || of_machine_is_compatible("PowerBook3,5") || @@ -710,9 +680,9 @@ out: pmac_cpu_freqs[CPUFREQ_HIGH].frequency = hi_freq; ppc_proc_freq = cur_freq * 1000ul; - printk(KERN_INFO "Registering PowerMac CPU frequency driver\n"); - printk(KERN_INFO "Low: %d Mhz, High: %d Mhz, Boot: %d Mhz\n", - low_freq/1000, hi_freq/1000, cur_freq/1000); + pr_info("Registering PowerMac CPU frequency driver\n"); + pr_info("Low: %d Mhz, High: %d Mhz, Boot: %d Mhz\n", + low_freq/1000, hi_freq/1000, cur_freq/1000); return cpufreq_register_driver(&pmac_cpufreq_driver); } diff --git a/drivers/cpufreq/pmac64-cpufreq.c b/drivers/cpufreq/pmac64-cpufreq.c index 7ba423431cfe..80897ec8f00e 100644 --- a/drivers/cpufreq/pmac64-cpufreq.c +++ b/drivers/cpufreq/pmac64-cpufreq.c @@ -1,17 +1,16 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2002 - 2005 Benjamin Herrenschmidt <benh@kernel.crashing.org> * and Markus Demleitner <msdemlei@cl.uni-heidelberg.de> * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * * This driver adds basic cpufreq support for SMU & 970FX based G5 Macs, * that is iMac G5 and latest single CPU desktop. */ #undef DEBUG +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + #include <linux/module.h> #include <linux/types.h> #include <linux/errno.h> @@ -22,7 +21,8 @@ #include <linux/init.h> #include <linux/completion.h> #include <linux/mutex.h> -#include <asm/prom.h> +#include <linux/of.h> + #include <asm/machdep.h> #include <asm/irq.h> #include <asm/sections.h> @@ -64,14 +64,9 @@ #define CPUFREQ_LOW 1 static struct cpufreq_frequency_table g5_cpu_freqs[] = { - {CPUFREQ_HIGH, 0}, - {CPUFREQ_LOW, 0}, - {0, CPUFREQ_TABLE_END}, -}; - -static struct freq_attr* g5_cpu_freqs_attr[] = { - &cpufreq_freq_attr_scaling_available_freqs, - NULL, + {0, CPUFREQ_HIGH, 0}, + {0, CPUFREQ_LOW, 0}, + {0, 0, CPUFREQ_TABLE_END}, }; /* Power mode data is an array of the 32 bits PCR values to use for @@ -83,8 +78,6 @@ static void (*g5_switch_volt)(int speed_mode); static int (*g5_switch_freq)(int speed_mode); static int (*g5_query_freq)(void); -static DEFINE_MUTEX(g5_switch_mutex); - static unsigned long transition_latency; #ifdef CONFIG_PMAC_SMU @@ -141,10 +134,10 @@ static void g5_vdnap_switch_volt(int speed_mode) pmf_call_one(pfunc_vdnap0_complete, &args); if (done) break; - msleep(1); + usleep_range(1000, 1000); } if (done == 0) - printk(KERN_WARNING "cpufreq: Timeout in clock slewing !\n"); + pr_warn("Timeout in clock slewing !\n"); } @@ -240,7 +233,7 @@ static void g5_pfunc_switch_volt(int speed_mode) if (pfunc_cpu1_volt_low) pmf_call_one(pfunc_cpu1_volt_low, NULL); } - msleep(10); /* should be faster , to fix */ + usleep_range(10000, 10000); /* should be faster , to fix */ } /* @@ -272,7 +265,7 @@ static int g5_pfunc_switch_freq(int speed_mode) rc = pmf_call_one(pfunc_cpu_setfreq_low, NULL); if (rc) - printk(KERN_WARNING "cpufreq: pfunc switch error %d\n", rc); + pr_warn("pfunc switch error %d\n", rc); /* It's an irq GPIO so we should be able to just block here, * I'll do that later after I've properly tested the IRQ code for @@ -285,10 +278,10 @@ static int g5_pfunc_switch_freq(int speed_mode) pmf_call_one(pfunc_slewing_done, &args); if (done) break; - msleep(1); + usleep_range(500, 500); } if (done == 0) - printk(KERN_WARNING "cpufreq: Timeout in clock slewing !\n"); + pr_warn("Timeout in clock slewing !\n"); /* If frequency is going down, last ramp the voltage */ if (speed_mode > g5_pmode_cur) @@ -316,37 +309,9 @@ static int g5_pfunc_query_freq(void) * Common interface to the cpufreq core */ -static int g5_cpufreq_verify(struct cpufreq_policy *policy) +static int g5_cpufreq_target(struct cpufreq_policy *policy, unsigned int index) { - return cpufreq_frequency_table_verify(policy, g5_cpu_freqs); -} - -static int g5_cpufreq_target(struct cpufreq_policy *policy, - unsigned int target_freq, unsigned int relation) -{ - unsigned int newstate = 0; - struct cpufreq_freqs freqs; - int rc; - - if (cpufreq_frequency_table_target(policy, g5_cpu_freqs, - target_freq, relation, &newstate)) - return -EINVAL; - - if (g5_pmode_cur == newstate) - return 0; - - mutex_lock(&g5_switch_mutex); - - freqs.old = g5_cpu_freqs[g5_pmode_cur].frequency; - freqs.new = g5_cpu_freqs[newstate].frequency; - - cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE); - rc = g5_switch_freq(newstate); - cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); - - mutex_unlock(&g5_switch_mutex); - - return rc; + return g5_switch_freq(index); } static unsigned int g5_cpufreq_get_speed(unsigned int cpu) @@ -356,36 +321,24 @@ static unsigned int g5_cpufreq_get_speed(unsigned int cpu) static int g5_cpufreq_cpu_init(struct cpufreq_policy *policy) { - policy->cpuinfo.transition_latency = transition_latency; - policy->cur = g5_cpu_freqs[g5_query_freq()].frequency; - /* secondary CPUs are tied to the primary one by the - * cpufreq core if in the secondary policy we tell it that - * it actually must be one policy together with all others. */ - cpumask_copy(policy->cpus, cpu_online_mask); - cpufreq_frequency_table_get_attr(g5_cpu_freqs, policy->cpu); - - return cpufreq_frequency_table_cpuinfo(policy, - g5_cpu_freqs); + cpufreq_generic_init(policy, g5_cpu_freqs, transition_latency); + return 0; } - static struct cpufreq_driver g5_cpufreq_driver = { .name = "powermac", - .owner = THIS_MODULE, .flags = CPUFREQ_CONST_LOOPS, .init = g5_cpufreq_cpu_init, - .verify = g5_cpufreq_verify, - .target = g5_cpufreq_target, + .verify = cpufreq_generic_frequency_table_verify, + .target_index = g5_cpufreq_target, .get = g5_cpufreq_get_speed, - .attr = g5_cpu_freqs_attr, }; #ifdef CONFIG_PMAC_SMU -static int __init g5_neo2_cpufreq_init(struct device_node *cpus) +static int __init g5_neo2_cpufreq_init(struct device_node *cpunode) { - struct device_node *cpunode; unsigned int psize, ssize; unsigned long max_freq; char *freq_method, *volt_method; @@ -398,27 +351,14 @@ static int __init g5_neo2_cpufreq_init(struct device_node *cpus) /* Check supported platforms */ if (of_machine_is_compatible("PowerMac8,1") || of_machine_is_compatible("PowerMac8,2") || - of_machine_is_compatible("PowerMac9,1")) + of_machine_is_compatible("PowerMac9,1") || + of_machine_is_compatible("PowerMac12,1")) use_volts_smu = 1; else if (of_machine_is_compatible("PowerMac11,2")) use_volts_vdnap = 1; else return -ENODEV; - /* Get first CPU node */ - for (cpunode = NULL; - (cpunode = of_get_next_child(cpus, cpunode)) != NULL;) { - const u32 *reg = of_get_property(cpunode, "reg", NULL); - if (reg == NULL || (*reg) != 0) - continue; - if (!strcmp(cpunode->type, "cpu")) - break; - } - if (cpunode == NULL) { - printk(KERN_ERR "cpufreq: Can't find any CPU 0 node\n"); - return -ENODEV; - } - /* Check 970FX for now */ valp = of_get_property(cpunode, "cpu-version", NULL); if (!valp) { @@ -427,7 +367,7 @@ static int __init g5_neo2_cpufreq_init(struct device_node *cpus) } pvr_hi = (*valp) >> 16; if (pvr_hi != 0x3c && pvr_hi != 0x44) { - printk(KERN_ERR "cpufreq: Unsupported CPU version\n"); + pr_err("Unsupported CPU version\n"); goto bail_noprops; } @@ -447,9 +387,8 @@ static int __init g5_neo2_cpufreq_init(struct device_node *cpus) if (!shdr) goto bail_noprops; g5_fvt_table = (struct smu_sdbp_fvt *)&shdr[1]; - ssize = (shdr->len * sizeof(u32)) - - sizeof(struct smu_sdbp_header); - g5_fvt_count = ssize / sizeof(struct smu_sdbp_fvt); + ssize = (shdr->len * sizeof(u32)) - sizeof(*shdr); + g5_fvt_count = ssize / sizeof(*g5_fvt_table); g5_fvt_cur = 0; /* Sanity checking */ @@ -463,17 +402,16 @@ static int __init g5_neo2_cpufreq_init(struct device_node *cpus) root = of_find_node_by_path("/"); if (root == NULL) { - printk(KERN_ERR "cpufreq: Can't find root of " - "device tree\n"); + pr_err("Can't find root of device tree\n"); goto bail_noprops; } pfunc_set_vdnap0 = pmf_find_function(root, "set-vdnap0"); pfunc_vdnap0_complete = pmf_find_function(root, "slewing-done"); + of_node_put(root); if (pfunc_set_vdnap0 == NULL || pfunc_vdnap0_complete == NULL) { - printk(KERN_ERR "cpufreq: Can't find required " - "platform function\n"); + pr_err("Can't find required platform function\n"); goto bail_noprops; } @@ -513,10 +451,10 @@ static int __init g5_neo2_cpufreq_init(struct device_node *cpus) g5_pmode_cur = -1; g5_switch_freq(g5_query_freq()); - printk(KERN_INFO "Registering G5 CPU frequency driver\n"); - printk(KERN_INFO "Frequency method: %s, Voltage method: %s\n", - freq_method, volt_method); - printk(KERN_INFO "Low: %d Mhz, High: %d Mhz, Cur: %d MHz\n", + pr_info("Registering G5 CPU frequency driver\n"); + pr_info("Frequency method: %s, Voltage method: %s\n", + freq_method, volt_method); + pr_info("Low: %d Mhz, High: %d Mhz, Cur: %d MHz\n", g5_cpu_freqs[1].frequency/1000, g5_cpu_freqs[0].frequency/1000, g5_cpu_freqs[g5_pmode_cur].frequency/1000); @@ -537,9 +475,9 @@ static int __init g5_neo2_cpufreq_init(struct device_node *cpus) #endif /* CONFIG_PMAC_SMU */ -static int __init g5_pm72_cpufreq_init(struct device_node *cpus) +static int __init g5_pm72_cpufreq_init(struct device_node *cpunode) { - struct device_node *cpuid = NULL, *hwclock = NULL, *cpunode = NULL; + struct device_node *cpuid = NULL, *hwclock = NULL; const u8 *eeprom = NULL; const u32 *valp; u64 max_freq, min_freq, ih, il; @@ -548,47 +486,35 @@ static int __init g5_pm72_cpufreq_init(struct device_node *cpus) DBG("cpufreq: Initializing for PowerMac7,2, PowerMac7,3 and" " RackMac3,1...\n"); - /* Get first CPU node */ - for (cpunode = NULL; - (cpunode = of_get_next_child(cpus, cpunode)) != NULL;) { - if (!strcmp(cpunode->type, "cpu")) - break; - } - if (cpunode == NULL) { - printk(KERN_ERR "cpufreq: Can't find any CPU node\n"); - return -ENODEV; - } - /* Lookup the cpuid eeprom node */ cpuid = of_find_node_by_path("/u3@0,f8000000/i2c@f8001000/cpuid@a0"); if (cpuid != NULL) eeprom = of_get_property(cpuid, "cpuid", NULL); if (eeprom == NULL) { - printk(KERN_ERR "cpufreq: Can't find cpuid EEPROM !\n"); + pr_err("Can't find cpuid EEPROM !\n"); rc = -ENODEV; goto bail; } /* Lookup the i2c hwclock */ - for (hwclock = NULL; - (hwclock = of_find_node_by_name(hwclock, "i2c-hwclock")) != NULL;){ + for_each_node_by_name(hwclock, "i2c-hwclock") { const char *loc = of_get_property(hwclock, "hwctrl-location", NULL); if (loc == NULL) continue; if (strcmp(loc, "CPU CLOCK")) continue; - if (!of_get_property(hwclock, "platform-get-frequency", NULL)) + if (!of_property_present(hwclock, "platform-get-frequency")) continue; break; } if (hwclock == NULL) { - printk(KERN_ERR "cpufreq: Can't find i2c clock chip !\n"); + pr_err("Can't find i2c clock chip !\n"); rc = -ENODEV; goto bail; } - DBG("cpufreq: i2c clock chip found: %s\n", hwclock->full_name); + DBG("cpufreq: i2c clock chip found: %pOF\n", hwclock); /* Now get all the platform functions */ pfunc_cpu_getfreq = @@ -611,7 +537,7 @@ static int __init g5_pm72_cpufreq_init(struct device_node *cpus) /* Check we have minimum requirements */ if (pfunc_cpu_getfreq == NULL || pfunc_cpu_setfreq_high == NULL || pfunc_cpu_setfreq_low == NULL || pfunc_slewing_done == NULL) { - printk(KERN_ERR "cpufreq: Can't find platform functions !\n"); + pr_err("Can't find platform functions !\n"); rc = -ENODEV; goto bail; } @@ -639,7 +565,7 @@ static int __init g5_pm72_cpufreq_init(struct device_node *cpus) /* Get max frequency from device-tree */ valp = of_get_property(cpunode, "clock-frequency", NULL); if (!valp) { - printk(KERN_ERR "cpufreq: Can't find CPU frequency !\n"); + pr_err("Can't find CPU frequency !\n"); rc = -ENODEV; goto bail; } @@ -655,8 +581,7 @@ static int __init g5_pm72_cpufreq_init(struct device_node *cpus) /* Check for machines with no useful settings */ if (il == ih) { - printk(KERN_WARNING "cpufreq: No low frequency mode available" - " on this model !\n"); + pr_warn("No low frequency mode available on this model !\n"); rc = -ENODEV; goto bail; } @@ -667,15 +592,17 @@ static int __init g5_pm72_cpufreq_init(struct device_node *cpus) /* Sanity check */ if (min_freq >= max_freq || min_freq < 1000) { - printk(KERN_ERR "cpufreq: Can't calculate low frequency !\n"); + pr_err("Can't calculate low frequency !\n"); rc = -ENXIO; goto bail; } g5_cpu_freqs[0].frequency = max_freq; g5_cpu_freqs[1].frequency = min_freq; + /* Based on a measurement on Xserve G5, rounded up. */ + transition_latency = 10 * NSEC_PER_MSEC; + /* Set callbacks */ - transition_latency = CPUFREQ_ETERNAL; g5_switch_volt = g5_pfunc_switch_volt; g5_switch_freq = g5_pfunc_switch_freq; g5_query_freq = g5_pfunc_query_freq; @@ -689,10 +616,10 @@ static int __init g5_pm72_cpufreq_init(struct device_node *cpus) g5_pmode_cur = -1; g5_switch_freq(g5_query_freq()); - printk(KERN_INFO "Registering G5 CPU frequency driver\n"); - printk(KERN_INFO "Frequency method: i2c/pfunc, " - "Voltage method: %s\n", has_volt ? "i2c/pfunc" : "none"); - printk(KERN_INFO "Low: %d Mhz, High: %d Mhz, Cur: %d MHz\n", + pr_info("Registering G5 CPU frequency driver\n"); + pr_info("Frequency method: i2c/pfunc, Voltage method: %s\n", + has_volt ? "i2c/pfunc" : "none"); + pr_info("Low: %d Mhz, High: %d Mhz, Cur: %d MHz\n", g5_cpu_freqs[1].frequency/1000, g5_cpu_freqs[0].frequency/1000, g5_cpu_freqs[g5_pmode_cur].frequency/1000); @@ -718,29 +645,30 @@ static int __init g5_pm72_cpufreq_init(struct device_node *cpus) static int __init g5_cpufreq_init(void) { - struct device_node *cpus; + struct device_node *cpunode; int rc = 0; - cpus = of_find_node_by_path("/cpus"); - if (cpus == NULL) { - DBG("No /cpus node !\n"); + /* Get first CPU node */ + cpunode = of_cpu_device_node_get(0); + if (cpunode == NULL) { + pr_err("Can't find any CPU node\n"); return -ENODEV; } if (of_machine_is_compatible("PowerMac7,2") || of_machine_is_compatible("PowerMac7,3") || of_machine_is_compatible("RackMac3,1")) - rc = g5_pm72_cpufreq_init(cpus); + rc = g5_pm72_cpufreq_init(cpunode); #ifdef CONFIG_PMAC_SMU else - rc = g5_neo2_cpufreq_init(cpus); + rc = g5_neo2_cpufreq_init(cpunode); #endif /* CONFIG_PMAC_SMU */ - of_node_put(cpus); return rc; } module_init(g5_cpufreq_init); +MODULE_DESCRIPTION("cpufreq driver for SMU & 970FX based G5 Macs"); MODULE_LICENSE("GPL"); diff --git a/drivers/cpufreq/powernow-k6.c b/drivers/cpufreq/powernow-k6.c index ea8e10382ec5..99d2244e03b0 100644 --- a/drivers/cpufreq/powernow-k6.c +++ b/drivers/cpufreq/powernow-k6.c @@ -1,13 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * This file was based upon code in Powertweak Linux (http://powertweak.sf.net) * (C) 2000-2003 Dave Jones, Arjan van de Ven, Janne Pänkälä, * Dominik Brodowski. * - * Licensed under the terms of the GNU GPL License version 2. - * * BIG FAT DISCLAIMER: Work in progress code. Possibly *dangerous* */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + #include <linux/kernel.h> #include <linux/module.h> #include <linux/init.h> @@ -22,170 +23,220 @@ #define POWERNOW_IOPORT 0xfff0 /* it doesn't matter where, as long as it is unused */ -#define PFX "powernow-k6: " static unsigned int busfreq; /* FSB, in 10 kHz */ static unsigned int max_multiplier; +static unsigned int param_busfreq = 0; +static unsigned int param_max_multiplier = 0; + +module_param_named(max_multiplier, param_max_multiplier, uint, S_IRUGO); +MODULE_PARM_DESC(max_multiplier, "Maximum multiplier (allowed values: 20 30 35 40 45 50 55 60)"); + +module_param_named(bus_frequency, param_busfreq, uint, S_IRUGO); +MODULE_PARM_DESC(bus_frequency, "Bus frequency in kHz"); /* Clock ratio multiplied by 10 - see table 27 in AMD#23446 */ static struct cpufreq_frequency_table clock_ratio[] = { - {45, /* 000 -> 4.5x */ 0}, - {50, /* 001 -> 5.0x */ 0}, - {40, /* 010 -> 4.0x */ 0}, - {55, /* 011 -> 5.5x */ 0}, - {20, /* 100 -> 2.0x */ 0}, - {30, /* 101 -> 3.0x */ 0}, - {60, /* 110 -> 6.0x */ 0}, - {35, /* 111 -> 3.5x */ 0}, - {0, CPUFREQ_TABLE_END} + {0, 60, /* 110 -> 6.0x */ 0}, + {0, 55, /* 011 -> 5.5x */ 0}, + {0, 50, /* 001 -> 5.0x */ 0}, + {0, 45, /* 000 -> 4.5x */ 0}, + {0, 40, /* 010 -> 4.0x */ 0}, + {0, 35, /* 111 -> 3.5x */ 0}, + {0, 30, /* 101 -> 3.0x */ 0}, + {0, 20, /* 100 -> 2.0x */ 0}, + {0, 0, CPUFREQ_TABLE_END} }; +static const u8 index_to_register[8] = { 6, 3, 1, 0, 2, 7, 5, 4 }; +static const u8 register_to_index[8] = { 3, 2, 4, 1, 7, 6, 0, 5 }; + +static const struct { + unsigned freq; + unsigned mult; +} usual_frequency_table[] = { + { 350000, 35 }, // 100 * 3.5 + { 400000, 40 }, // 100 * 4 + { 450000, 45 }, // 100 * 4.5 + { 475000, 50 }, // 95 * 5 + { 500000, 50 }, // 100 * 5 + { 506250, 45 }, // 112.5 * 4.5 + { 533500, 55 }, // 97 * 5.5 + { 550000, 55 }, // 100 * 5.5 + { 562500, 50 }, // 112.5 * 5 + { 570000, 60 }, // 95 * 6 + { 600000, 60 }, // 100 * 6 + { 618750, 55 }, // 112.5 * 5.5 + { 660000, 55 }, // 120 * 5.5 + { 675000, 60 }, // 112.5 * 6 + { 720000, 60 }, // 120 * 6 +}; + +#define FREQ_RANGE 3000 /** * powernow_k6_get_cpu_multiplier - returns the current FSB multiplier * - * Returns the current setting of the frequency multiplier. Core clock + * Returns the current setting of the frequency multiplier. Core clock * speed is frequency of the Front-Side Bus multiplied with this value. */ static int powernow_k6_get_cpu_multiplier(void) { - u64 invalue = 0; + unsigned long invalue = 0; u32 msrval; + local_irq_disable(); + msrval = POWERNOW_IOPORT + 0x1; wrmsr(MSR_K6_EPMR, msrval, 0); /* enable the PowerNow port */ invalue = inl(POWERNOW_IOPORT + 0x8); msrval = POWERNOW_IOPORT + 0x0; wrmsr(MSR_K6_EPMR, msrval, 0); /* disable it again */ - return clock_ratio[(invalue >> 5)&7].driver_data; -} + local_irq_enable(); + return clock_ratio[register_to_index[(invalue >> 5)&7]].driver_data; +} -/** - * powernow_k6_set_state - set the PowerNow! multiplier - * @best_i: clock_ratio[best_i] is the target multiplier - * - * Tries to change the PowerNow! multiplier - */ -static void powernow_k6_set_state(struct cpufreq_policy *policy, - unsigned int best_i) +static void powernow_k6_set_cpu_multiplier(unsigned int best_i) { - unsigned long outvalue = 0, invalue = 0; + unsigned long outvalue, invalue; unsigned long msrval; - struct cpufreq_freqs freqs; - - if (clock_ratio[best_i].driver_data > max_multiplier) { - printk(KERN_ERR PFX "invalid target frequency\n"); - return; - } - - freqs.old = busfreq * powernow_k6_get_cpu_multiplier(); - freqs.new = busfreq * clock_ratio[best_i].driver_data; - - cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE); + unsigned long cr0; /* we now need to transform best_i to the BVC format, see AMD#23446 */ - outvalue = (1<<12) | (1<<10) | (1<<9) | (best_i<<5); + /* + * The processor doesn't respond to inquiry cycles while changing the + * frequency, so we must disable cache. + */ + local_irq_disable(); + cr0 = read_cr0(); + write_cr0(cr0 | X86_CR0_CD); + wbinvd(); + + outvalue = (1<<12) | (1<<10) | (1<<9) | (index_to_register[best_i]<<5); msrval = POWERNOW_IOPORT + 0x1; wrmsr(MSR_K6_EPMR, msrval, 0); /* enable the PowerNow port */ invalue = inl(POWERNOW_IOPORT + 0x8); - invalue = invalue & 0xf; + invalue = invalue & 0x1f; outvalue = outvalue | invalue; - outl(outvalue , (POWERNOW_IOPORT + 0x8)); + outl(outvalue, (POWERNOW_IOPORT + 0x8)); msrval = POWERNOW_IOPORT + 0x0; wrmsr(MSR_K6_EPMR, msrval, 0); /* disable it again */ - cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); - - return; -} - - -/** - * powernow_k6_verify - verifies a new CPUfreq policy - * @policy: new policy - * - * Policy must be within lowest and highest possible CPU Frequency, - * and at least one possible state must be within min and max. - */ -static int powernow_k6_verify(struct cpufreq_policy *policy) -{ - return cpufreq_frequency_table_verify(policy, &clock_ratio[0]); + write_cr0(cr0); + local_irq_enable(); } - /** - * powernow_k6_setpolicy - sets a new CPUFreq policy - * @policy: new policy - * @target_freq: the target frequency - * @relation: how that frequency relates to achieved frequency - * (CPUFREQ_RELATION_L or CPUFREQ_RELATION_H) + * powernow_k6_target - set the PowerNow! multiplier + * @best_i: clock_ratio[best_i] is the target multiplier * - * sets a new CPUFreq policy + * Tries to change the PowerNow! multiplier */ static int powernow_k6_target(struct cpufreq_policy *policy, - unsigned int target_freq, - unsigned int relation) + unsigned int best_i) { - unsigned int newstate = 0; - if (cpufreq_frequency_table_target(policy, &clock_ratio[0], - target_freq, relation, &newstate)) + if (clock_ratio[best_i].driver_data > max_multiplier) { + pr_err("invalid target frequency\n"); return -EINVAL; + } - powernow_k6_set_state(policy, newstate); + powernow_k6_set_cpu_multiplier(best_i); return 0; } - static int powernow_k6_cpu_init(struct cpufreq_policy *policy) { + struct cpufreq_frequency_table *pos; unsigned int i, f; - int result; + unsigned khz; if (policy->cpu != 0) return -ENODEV; - /* get frequencies */ - max_multiplier = powernow_k6_get_cpu_multiplier(); - busfreq = cpu_khz / max_multiplier; + max_multiplier = 0; + khz = cpu_khz; + for (i = 0; i < ARRAY_SIZE(usual_frequency_table); i++) { + if (khz >= usual_frequency_table[i].freq - FREQ_RANGE && + khz <= usual_frequency_table[i].freq + FREQ_RANGE) { + khz = usual_frequency_table[i].freq; + max_multiplier = usual_frequency_table[i].mult; + break; + } + } + if (param_max_multiplier) { + cpufreq_for_each_entry(pos, clock_ratio) + if (pos->driver_data == param_max_multiplier) { + max_multiplier = param_max_multiplier; + goto have_max_multiplier; + } + pr_err("invalid max_multiplier parameter, valid parameters 20, 30, 35, 40, 45, 50, 55, 60\n"); + return -EINVAL; + } + + if (!max_multiplier) { + pr_warn("unknown frequency %u, cannot determine current multiplier\n", + khz); + pr_warn("use module parameters max_multiplier and bus_frequency\n"); + return -EOPNOTSUPP; + } + +have_max_multiplier: + param_max_multiplier = max_multiplier; + + if (param_busfreq) { + if (param_busfreq >= 50000 && param_busfreq <= 150000) { + busfreq = param_busfreq / 10; + goto have_busfreq; + } + pr_err("invalid bus_frequency parameter, allowed range 50000 - 150000 kHz\n"); + return -EINVAL; + } + + busfreq = khz / max_multiplier; +have_busfreq: + param_busfreq = busfreq * 10; /* table init */ - for (i = 0; (clock_ratio[i].frequency != CPUFREQ_TABLE_END); i++) { - f = clock_ratio[i].driver_data; + cpufreq_for_each_entry(pos, clock_ratio) { + f = pos->driver_data; if (f > max_multiplier) - clock_ratio[i].frequency = CPUFREQ_ENTRY_INVALID; + pos->frequency = CPUFREQ_ENTRY_INVALID; else - clock_ratio[i].frequency = busfreq * f; + pos->frequency = busfreq * f; } /* cpuinfo and default policy values */ - policy->cpuinfo.transition_latency = 200000; - policy->cur = busfreq * max_multiplier; - - result = cpufreq_frequency_table_cpuinfo(policy, clock_ratio); - if (result) - return result; - - cpufreq_frequency_table_get_attr(clock_ratio, policy->cpu); + policy->cpuinfo.transition_latency = 500000; + policy->freq_table = clock_ratio; return 0; } -static int powernow_k6_cpu_exit(struct cpufreq_policy *policy) +static void powernow_k6_cpu_exit(struct cpufreq_policy *policy) { unsigned int i; - for (i = 0; i < 8; i++) { - if (i == max_multiplier) - powernow_k6_set_state(policy, i); + + for (i = 0; (clock_ratio[i].frequency != CPUFREQ_TABLE_END); i++) { + if (clock_ratio[i].driver_data == max_multiplier) { + struct cpufreq_freqs freqs; + + freqs.old = policy->cur; + freqs.new = clock_ratio[i].frequency; + freqs.flags = 0; + + cpufreq_freq_transition_begin(policy, &freqs); + powernow_k6_target(policy, i); + cpufreq_freq_transition_end(policy, &freqs, 0); + return; + } } - cpufreq_frequency_table_put_attr(policy->cpu); - return 0; } static unsigned int powernow_k6_get(unsigned int cpu) @@ -195,25 +246,18 @@ static unsigned int powernow_k6_get(unsigned int cpu) return ret; } -static struct freq_attr *powernow_k6_attr[] = { - &cpufreq_freq_attr_scaling_available_freqs, - NULL, -}; - static struct cpufreq_driver powernow_k6_driver = { - .verify = powernow_k6_verify, - .target = powernow_k6_target, + .verify = cpufreq_generic_frequency_table_verify, + .target_index = powernow_k6_target, .init = powernow_k6_cpu_init, .exit = powernow_k6_cpu_exit, .get = powernow_k6_get, .name = "powernow-k6", - .owner = THIS_MODULE, - .attr = powernow_k6_attr, }; static const struct x86_cpu_id powernow_k6_ids[] = { - { X86_VENDOR_AMD, 5, 12 }, - { X86_VENDOR_AMD, 5, 13 }, + X86_MATCH_VENDOR_FAM_MODEL(AMD, 5, 12, NULL), + X86_MATCH_VENDOR_FAM_MODEL(AMD, 5, 13, NULL), {} }; MODULE_DEVICE_TABLE(x86cpu, powernow_k6_ids); @@ -231,7 +275,7 @@ static int __init powernow_k6_init(void) return -ENODEV; if (!request_region(POWERNOW_IOPORT, 16, "PowerNow!")) { - printk(KERN_INFO PFX "PowerNow IOPORT region already used.\n"); + pr_info("PowerNow IOPORT region already used\n"); return -EIO; } @@ -256,7 +300,7 @@ static void __exit powernow_k6_exit(void) } -MODULE_AUTHOR("Arjan van de Ven, Dave Jones <davej@redhat.com>, " +MODULE_AUTHOR("Arjan van de Ven, Dave Jones, " "Dominik Brodowski <linux@brodo.de>"); MODULE_DESCRIPTION("PowerNow! driver for AMD K6-2+ / K6-3+ processors."); MODULE_LICENSE("GPL"); diff --git a/drivers/cpufreq/powernow-k7.c b/drivers/cpufreq/powernow-k7.c index b9f80b713fda..31039330a3ba 100644 --- a/drivers/cpufreq/powernow-k7.c +++ b/drivers/cpufreq/powernow-k7.c @@ -1,9 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * AMD K7 Powernow driver. * (C) 2003 Dave Jones on behalf of SuSE Labs. - * (C) 2003-2004 Dave Jones <davej@redhat.com> * - * Licensed under the terms of the GNU GPL License version 2. * Based upon datasheets & sample CPUs kindly provided by AMD. * * Errata 5: @@ -14,6 +13,8 @@ * - We disable half multipliers if ACPI is used on A0 stepping CPUs. */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + #include <linux/kernel.h> #include <linux/module.h> #include <linux/moduleparam.h> @@ -36,9 +37,6 @@ #include "powernow-k7.h" -#define PFX "powernow: " - - struct psb_s { u8 signature[10]; u8 tableversion; @@ -111,7 +109,7 @@ static int check_fsb(unsigned int fsbspeed) } static const struct x86_cpu_id powernow_k7_cpuids[] = { - { X86_VENDOR_AMD, 6, }, + X86_MATCH_VENDOR_FAM(AMD, 6, NULL), {} }; MODULE_DEVICE_TABLE(x86cpu, powernow_k7_cpuids); @@ -128,14 +126,13 @@ static int check_powernow(void) maxei = cpuid_eax(0x80000000); if (maxei < 0x80000007) { /* Any powernow info ? */ #ifdef MODULE - printk(KERN_INFO PFX "No powernow capabilities detected\n"); + pr_info("No powernow capabilities detected\n"); #endif return 0; } - if ((c->x86_model == 6) && (c->x86_mask == 0)) { - printk(KERN_INFO PFX "K7 660[A0] core detected, " - "enabling errata workarounds\n"); + if ((c->x86_model == 6) && (c->x86_stepping == 0)) { + pr_info("K7 660[A0] core detected, enabling errata workarounds\n"); have_a0 = 1; } @@ -145,22 +142,22 @@ static int check_powernow(void) if (!(edx & (1 << 1 | 1 << 2))) return 0; - printk(KERN_INFO PFX "PowerNOW! Technology present. Can scale: "); + pr_info("PowerNOW! Technology present. Can scale: "); if (edx & 1 << 1) { - printk("frequency"); + pr_cont("frequency"); can_scale_bus = 1; } if ((edx & (1 << 1 | 1 << 2)) == 0x6) - printk(" and "); + pr_cont(" and "); if (edx & 1 << 2) { - printk("voltage"); + pr_cont("voltage"); can_scale_vid = 1; } - printk(".\n"); + pr_cont("\n"); return 1; } @@ -177,7 +174,7 @@ static int get_ranges(unsigned char *pst) unsigned int speed; u8 fid, vid; - powernow_table = kzalloc((sizeof(struct cpufreq_frequency_table) * + powernow_table = kzalloc((sizeof(*powernow_table) * (number_scales + 1)), GFP_KERNEL); if (!powernow_table) return -ENOMEM; @@ -222,13 +219,13 @@ static void change_FID(int fid) { union msr_fidvidctl fidvidctl; - rdmsrl(MSR_K7_FID_VID_CTL, fidvidctl.val); + rdmsrq(MSR_K7_FID_VID_CTL, fidvidctl.val); if (fidvidctl.bits.FID != fid) { fidvidctl.bits.SGTC = latency; fidvidctl.bits.FID = fid; fidvidctl.bits.VIDC = 0; fidvidctl.bits.FIDC = 1; - wrmsrl(MSR_K7_FID_VID_CTL, fidvidctl.val); + wrmsrq(MSR_K7_FID_VID_CTL, fidvidctl.val); } } @@ -237,18 +234,18 @@ static void change_VID(int vid) { union msr_fidvidctl fidvidctl; - rdmsrl(MSR_K7_FID_VID_CTL, fidvidctl.val); + rdmsrq(MSR_K7_FID_VID_CTL, fidvidctl.val); if (fidvidctl.bits.VID != vid) { fidvidctl.bits.SGTC = latency; fidvidctl.bits.VID = vid; fidvidctl.bits.FIDC = 0; fidvidctl.bits.VIDC = 1; - wrmsrl(MSR_K7_FID_VID_CTL, fidvidctl.val); + wrmsrq(MSR_K7_FID_VID_CTL, fidvidctl.val); } } -static void change_speed(struct cpufreq_policy *policy, unsigned int index) +static int powernow_target(struct cpufreq_policy *policy, unsigned int index) { u8 fid, vid; struct cpufreq_freqs freqs; @@ -263,14 +260,12 @@ static void change_speed(struct cpufreq_policy *policy, unsigned int index) fid = powernow_table[index].driver_data & 0xFF; vid = (powernow_table[index].driver_data & 0xFF00) >> 8; - rdmsrl(MSR_K7_FID_VID_STATUS, fidvidstatus.val); + rdmsrq(MSR_K7_FID_VID_STATUS, fidvidstatus.val); cfid = fidvidstatus.bits.CFID; freqs.old = fsb * fid_codes[cfid] / 10; freqs.new = powernow_table[index].frequency; - cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE); - /* Now do the magic poking into the MSRs. */ if (have_a0 == 1) /* A0 errata 5 */ @@ -290,7 +285,7 @@ static void change_speed(struct cpufreq_policy *policy, unsigned int index) if (have_a0 == 1) local_irq_enable(); - cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); + return 0; } @@ -309,8 +304,7 @@ static int powernow_acpi_init(void) goto err0; } - acpi_processor_perf = kzalloc(sizeof(struct acpi_processor_performance), - GFP_KERNEL); + acpi_processor_perf = kzalloc(sizeof(*acpi_processor_perf), GFP_KERNEL); if (!acpi_processor_perf) { retval = -ENOMEM; goto err0; @@ -346,7 +340,7 @@ static int powernow_acpi_init(void) goto err2; } - powernow_table = kzalloc((sizeof(struct cpufreq_frequency_table) * + powernow_table = kzalloc((sizeof(*powernow_table) * (number_scales + 1)), GFP_KERNEL); if (!powernow_table) { retval = -ENOMEM; @@ -425,22 +419,20 @@ static int powernow_acpi_init(void) return 0; err2: - acpi_processor_unregister_performance(acpi_processor_perf, 0); + acpi_processor_unregister_performance(0); err1: free_cpumask_var(acpi_processor_perf->shared_cpu_map); err05: kfree(acpi_processor_perf); err0: - printk(KERN_WARNING PFX "ACPI perflib can not be used on " - "this platform\n"); + pr_warn("ACPI perflib can not be used on this platform\n"); acpi_processor_perf = NULL; return retval; } #else static int powernow_acpi_init(void) { - printk(KERN_INFO PFX "no support for ACPI processor found." - " Please recompile your kernel with ACPI processor\n"); + pr_info("no support for ACPI processor found - please recompile your kernel with ACPI processor\n"); return -EINVAL; } #endif @@ -472,8 +464,7 @@ static int powernow_decode_bios(int maxfid, int startvid) psb = (struct psb_s *) p; pr_debug("Table version: 0x%x\n", psb->tableversion); if (psb->tableversion != 0x12) { - printk(KERN_INFO PFX "Sorry, only v1.2 tables" - " supported right now\n"); + pr_info("Sorry, only v1.2 tables supported right now\n"); return -ENODEV; } @@ -485,10 +476,8 @@ static int powernow_decode_bios(int maxfid, int startvid) latency = psb->settlingtime; if (latency < 100) { - printk(KERN_INFO PFX "BIOS set settling time " - "to %d microseconds. " - "Should be at least 100. " - "Correcting.\n", latency); + pr_info("BIOS set settling time to %d microseconds. Should be at least 100. Correcting.\n", + latency); latency = 100; } pr_debug("Settling Time: %d microseconds.\n", @@ -497,7 +486,7 @@ static int powernow_decode_bios(int maxfid, int startvid) "relevant to this CPU).\n", psb->numpst); - p += sizeof(struct psb_s); + p += sizeof(*psb); pst = (struct pst_s *) p; @@ -510,20 +499,19 @@ static int powernow_decode_bios(int maxfid, int startvid) (maxfid == pst->maxfid) && (startvid == pst->startvid)) { print_pst_entry(pst, j); - p = (char *)pst + sizeof(struct pst_s); + p = (char *)pst + sizeof(*pst); ret = get_ranges(p); return ret; } else { unsigned int k; - p = (char *)pst + sizeof(struct pst_s); + p = (char *)pst + sizeof(*pst); for (k = 0; k < number_scales; k++) p += 2; } } - printk(KERN_INFO PFX "No PST tables match this cpuid " - "(0x%x)\n", etuple); - printk(KERN_INFO PFX "This is indicative of a broken " - "BIOS.\n"); + pr_info("No PST tables match this cpuid (0x%x)\n", + etuple); + pr_info("This is indicative of a broken BIOS\n"); return -EINVAL; } @@ -534,27 +522,6 @@ static int powernow_decode_bios(int maxfid, int startvid) } -static int powernow_target(struct cpufreq_policy *policy, - unsigned int target_freq, - unsigned int relation) -{ - unsigned int newstate; - - if (cpufreq_frequency_table_target(policy, powernow_table, target_freq, - relation, &newstate)) - return -EINVAL; - - change_speed(policy, newstate); - - return 0; -} - - -static int powernow_verify(struct cpufreq_policy *policy) -{ - return cpufreq_frequency_table_verify(policy, powernow_table); -} - /* * We use the fact that the bus frequency is somehow * a multiple of 100000/3 khz, then we compute sgtc according @@ -563,7 +530,7 @@ static int powernow_verify(struct cpufreq_policy *policy) * We will then get the same kind of behaviour already tested under * the "well-known" other OS. */ -static int __cpuinit fixup_sgtc(void) +static int fixup_sgtc(void) { unsigned int sgtc; unsigned int m; @@ -577,7 +544,7 @@ static int __cpuinit fixup_sgtc(void) sgtc = 100 * m * latency; sgtc = sgtc / 3; if (sgtc > 0xfffff) { - printk(KERN_WARNING PFX "SGTC too large %d\n", sgtc); + pr_warn("SGTC too large %d\n", sgtc); sgtc = 0xfffff; } return sgtc; @@ -590,23 +557,19 @@ static unsigned int powernow_get(unsigned int cpu) if (cpu) return 0; - rdmsrl(MSR_K7_FID_VID_STATUS, fidvidstatus.val); + rdmsrq(MSR_K7_FID_VID_STATUS, fidvidstatus.val); cfid = fidvidstatus.bits.CFID; return fsb * fid_codes[cfid] / 10; } -static int __cpuinit acer_cpufreq_pst(const struct dmi_system_id *d) +static int acer_cpufreq_pst(const struct dmi_system_id *d) { - printk(KERN_WARNING PFX - "%s laptop with broken PST tables in BIOS detected.\n", + pr_warn("%s laptop with broken PST tables in BIOS detected\n", d->ident); - printk(KERN_WARNING PFX - "You need to downgrade to 3A21 (09/09/2002), or try a newer " - "BIOS than 3A71 (01/20/2003)\n"); - printk(KERN_WARNING PFX - "cpufreq scaling has been disabled as a result of this.\n"); + pr_warn("You need to downgrade to 3A21 (09/09/2002), or try a newer BIOS than 3A71 (01/20/2003)\n"); + pr_warn("cpufreq scaling has been disabled as a result of this\n"); return 0; } @@ -615,7 +578,7 @@ static int __cpuinit acer_cpufreq_pst(const struct dmi_system_id *d) * A BIOS update is all that can save them. * Mention this, and disable cpufreq. */ -static struct dmi_system_id __cpuinitdata powernow_dmi_table[] = { +static const struct dmi_system_id powernow_dmi_table[] = { { .callback = acer_cpufreq_pst, .ident = "Acer Aspire", @@ -627,7 +590,7 @@ static struct dmi_system_id __cpuinitdata powernow_dmi_table[] = { { } }; -static int __cpuinit powernow_cpu_init(struct cpufreq_policy *policy) +static int powernow_cpu_init(struct cpufreq_policy *policy) { union msr_fidvidstatus fidvidstatus; int result; @@ -635,81 +598,68 @@ static int __cpuinit powernow_cpu_init(struct cpufreq_policy *policy) if (policy->cpu != 0) return -ENODEV; - rdmsrl(MSR_K7_FID_VID_STATUS, fidvidstatus.val); + rdmsrq(MSR_K7_FID_VID_STATUS, fidvidstatus.val); recalibrate_cpu_khz(); fsb = (10 * cpu_khz) / fid_codes[fidvidstatus.bits.CFID]; if (!fsb) { - printk(KERN_WARNING PFX "can not determine bus frequency\n"); + pr_warn("can not determine bus frequency\n"); return -EINVAL; } pr_debug("FSB: %3dMHz\n", fsb/1000); if (dmi_check_system(powernow_dmi_table) || acpi_force) { - printk(KERN_INFO PFX "PSB/PST known to be broken. " - "Trying ACPI instead\n"); + pr_info("PSB/PST known to be broken - trying ACPI instead\n"); result = powernow_acpi_init(); } else { result = powernow_decode_bios(fidvidstatus.bits.MFID, fidvidstatus.bits.SVID); if (result) { - printk(KERN_INFO PFX "Trying ACPI perflib\n"); + pr_info("Trying ACPI perflib\n"); maximum_speed = 0; minimum_speed = -1; latency = 0; result = powernow_acpi_init(); if (result) { - printk(KERN_INFO PFX - "ACPI and legacy methods failed\n"); + pr_info("ACPI and legacy methods failed\n"); } } else { /* SGTC use the bus clock as timer */ latency = fixup_sgtc(); - printk(KERN_INFO PFX "SGTC: %d\n", latency); + pr_info("SGTC: %d\n", latency); } } if (result) return result; - printk(KERN_INFO PFX "Minimum speed %d MHz. Maximum speed %d MHz.\n", - minimum_speed/1000, maximum_speed/1000); + pr_info("Minimum speed %d MHz - Maximum speed %d MHz\n", + minimum_speed/1000, maximum_speed/1000); policy->cpuinfo.transition_latency = cpufreq_scale(2000000UL, fsb, latency); + policy->freq_table = powernow_table; - policy->cur = powernow_get(0); - - cpufreq_frequency_table_get_attr(powernow_table, policy->cpu); - - return cpufreq_frequency_table_cpuinfo(policy, powernow_table); + return 0; } -static int powernow_cpu_exit(struct cpufreq_policy *policy) +static void powernow_cpu_exit(struct cpufreq_policy *policy) { - cpufreq_frequency_table_put_attr(policy->cpu); - #ifdef CONFIG_X86_POWERNOW_K7_ACPI if (acpi_processor_perf) { - acpi_processor_unregister_performance(acpi_processor_perf, 0); + acpi_processor_unregister_performance(0); free_cpumask_var(acpi_processor_perf->shared_cpu_map); kfree(acpi_processor_perf); } #endif kfree(powernow_table); - return 0; } -static struct freq_attr *powernow_table_attr[] = { - &cpufreq_freq_attr_scaling_available_freqs, - NULL, -}; - static struct cpufreq_driver powernow_driver = { - .verify = powernow_verify, - .target = powernow_target, + .verify = cpufreq_generic_frequency_table_verify, + .target_index = powernow_target, .get = powernow_get, #ifdef CONFIG_X86_POWERNOW_K7_ACPI .bios_limit = acpi_processor_get_bios_limit, @@ -717,8 +667,6 @@ static struct cpufreq_driver powernow_driver = { .init = powernow_cpu_init, .exit = powernow_cpu_exit, .name = "powernow-k7", - .owner = THIS_MODULE, - .attr = powernow_table_attr, }; static int __init powernow_init(void) @@ -737,7 +685,7 @@ static void __exit powernow_exit(void) module_param(acpi_force, int, 0444); MODULE_PARM_DESC(acpi_force, "Force ACPI to be used."); -MODULE_AUTHOR("Dave Jones <davej@redhat.com>"); +MODULE_AUTHOR("Dave Jones"); MODULE_DESCRIPTION("Powernow driver for AMD K7 processors."); MODULE_LICENSE("GPL"); diff --git a/drivers/cpufreq/powernow-k7.h b/drivers/cpufreq/powernow-k7.h index 35fb4eaf6e1c..4bc673faeef7 100644 --- a/drivers/cpufreq/powernow-k7.h +++ b/drivers/cpufreq/powernow-k7.h @@ -1,10 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * (C) 2003 Dave Jones. * - * Licensed under the terms of the GNU GPL License version 2. - * * AMD-specific information - * */ union msr_fidvidctl { diff --git a/drivers/cpufreq/powernow-k8.c b/drivers/cpufreq/powernow-k8.c index 78f018f2a5de..f7512b4e923e 100644 --- a/drivers/cpufreq/powernow-k8.c +++ b/drivers/cpufreq/powernow-k8.c @@ -1,8 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * (c) 2003-2012 Advanced Micro Devices, Inc. - * Your use of this code is subject to the terms and conditions of the - * GNU general public license version 2. See "COPYING" or - * http://www.gnu.org/licenses/gpl.html * * Maintainer: * Andreas Herrmann <herrmann.der.user@googlemail.com> @@ -11,7 +9,6 @@ * (C) 2003 Dave Jones on behalf of SuSE Labs * (C) 2004 Dominik Brodowski <linux@brodo.de> * (C) 2004 Pavel Machek <pavel@ucw.cz> - * Licensed under the terms of the GNU GPL License version 2. * Based upon datasheets & sample CPUs kindly provided by AMD. * * Valuable input gratefully received from Dave Jones, Pavel Machek, @@ -27,6 +24,8 @@ * power and thermal data sheets, (e.g. 30417.pdf, 30430.pdf, 43375.pdf) */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + #include <linux/kernel.h> #include <linux/smp.h> #include <linux/module.h> @@ -45,7 +44,6 @@ #include <linux/mutex.h> #include <acpi/processor.h> -#define PFX "powernow-k8: " #define VERSION "version 2.20.00" #include "powernow-k8.h" @@ -56,13 +54,6 @@ static DEFINE_PER_CPU(struct powernow_k8_data *, powernow_data); static struct cpufreq_driver cpufreq_amd64_driver; -#ifndef CONFIG_SMP -static inline const struct cpumask *cpu_core_mask(int cpu) -{ - return cpumask_of(0); -} -#endif - /* Return a frequency in MHz, given an input fid */ static u32 find_freq_from_fid(u32 fid) { @@ -95,7 +86,7 @@ static u32 convert_fid_to_vco_fid(u32 fid) */ static int pending_bit_stuck(void) { - u32 lo, hi; + u32 lo, hi __always_unused; rdmsr(MSR_FIDVID_STATUS, lo, hi); return lo & MSR_S_LO_CHANGE_PENDING ? 1 : 0; @@ -128,14 +119,12 @@ static int query_current_values_with_pending_wait(struct powernow_k8_data *data) static void count_off_irt(struct powernow_k8_data *data) { udelay((1 << data->irt) * 10); - return; } /* the voltage stabilization time */ static void count_off_vst(struct powernow_k8_data *data) { udelay(data->vstable * VST_UNITS_20US); - return; } /* need to init the control msr to a safe value (for each cpu) */ @@ -161,7 +150,7 @@ static int write_new_fid(struct powernow_k8_data *data, u32 fid) u32 i = 0; if ((fid & INVALID_FID_MASK) || (data->currvid & INVALID_VID_MASK)) { - printk(KERN_ERR PFX "internal error - overflow on fid write\n"); + pr_err("internal error - overflow on fid write\n"); return 1; } @@ -175,9 +164,7 @@ static int write_new_fid(struct powernow_k8_data *data, u32 fid) do { wrmsr(MSR_FIDVID_CTL, lo, data->plllock * PLL_LOCK_CONVERSION); if (i++ > 100) { - printk(KERN_ERR PFX - "Hardware error - pending bit very stuck - " - "no further pstate changes possible\n"); + pr_err("Hardware error - pending bit very stuck - no further pstate changes possible\n"); return 1; } } while (query_current_values_with_pending_wait(data)); @@ -185,15 +172,13 @@ static int write_new_fid(struct powernow_k8_data *data, u32 fid) count_off_irt(data); if (savevid != data->currvid) { - printk(KERN_ERR PFX - "vid change on fid trans, old 0x%x, new 0x%x\n", - savevid, data->currvid); + pr_err("vid change on fid trans, old 0x%x, new 0x%x\n", + savevid, data->currvid); return 1; } if (fid != data->currfid) { - printk(KERN_ERR PFX - "fid trans failed, fid 0x%x, curr 0x%x\n", fid, + pr_err("fid trans failed, fid 0x%x, curr 0x%x\n", fid, data->currfid); return 1; } @@ -209,7 +194,7 @@ static int write_new_vid(struct powernow_k8_data *data, u32 vid) int i = 0; if ((data->currfid & INVALID_FID_MASK) || (vid & INVALID_VID_MASK)) { - printk(KERN_ERR PFX "internal error - overflow on vid write\n"); + pr_err("internal error - overflow on vid write\n"); return 1; } @@ -223,23 +208,19 @@ static int write_new_vid(struct powernow_k8_data *data, u32 vid) do { wrmsr(MSR_FIDVID_CTL, lo, STOP_GRANT_5NS); if (i++ > 100) { - printk(KERN_ERR PFX "internal error - pending bit " - "very stuck - no further pstate " - "changes possible\n"); + pr_err("internal error - pending bit very stuck - no further pstate changes possible\n"); return 1; } } while (query_current_values_with_pending_wait(data)); if (savefid != data->currfid) { - printk(KERN_ERR PFX "fid changed on vid trans, old " - "0x%x new 0x%x\n", - savefid, data->currfid); + pr_err("fid changed on vid trans, old 0x%x new 0x%x\n", + savefid, data->currfid); return 1; } if (vid != data->currvid) { - printk(KERN_ERR PFX "vid trans failed, vid 0x%x, " - "curr 0x%x\n", + pr_err("vid trans failed, vid 0x%x, curr 0x%x\n", vid, data->currvid); return 1; } @@ -283,8 +264,7 @@ static int transition_fid_vid(struct powernow_k8_data *data, return 1; if ((reqfid != data->currfid) || (reqvid != data->currvid)) { - printk(KERN_ERR PFX "failed (cpu%d): req 0x%x 0x%x, " - "curr 0x%x 0x%x\n", + pr_err("failed (cpu%d): req 0x%x 0x%x, curr 0x%x 0x%x\n", smp_processor_id(), reqfid, reqvid, data->currfid, data->currvid); return 1; @@ -302,10 +282,9 @@ static int core_voltage_pre_transition(struct powernow_k8_data *data, { u32 rvosteps = data->rvo; u32 savefid = data->currfid; - u32 maxvid, lo, rvomult = 1; + u32 maxvid, lo __always_unused, rvomult = 1; - pr_debug("ph1 (cpu%d): start, currfid 0x%x, currvid 0x%x, " - "reqvid 0x%x, rvo 0x%x\n", + pr_debug("ph1 (cpu%d): start, currfid 0x%x, currvid 0x%x, reqvid 0x%x, rvo 0x%x\n", smp_processor_id(), data->currfid, data->currvid, reqvid, data->rvo); @@ -342,8 +321,7 @@ static int core_voltage_pre_transition(struct powernow_k8_data *data, return 1; if (savefid != data->currfid) { - printk(KERN_ERR PFX "ph1 err, currfid changed 0x%x\n", - data->currfid); + pr_err("ph1 err, currfid changed 0x%x\n", data->currfid); return 1; } @@ -360,13 +338,11 @@ static int core_frequency_transition(struct powernow_k8_data *data, u32 reqfid) u32 fid_interval, savevid = data->currvid; if (data->currfid == reqfid) { - printk(KERN_ERR PFX "ph2 null fid transition 0x%x\n", - data->currfid); + pr_err("ph2 null fid transition 0x%x\n", data->currfid); return 0; } - pr_debug("ph2 (cpu%d): starting, currfid 0x%x, currvid 0x%x, " - "reqfid 0x%x\n", + pr_debug("ph2 (cpu%d): starting, currfid 0x%x, currvid 0x%x, reqfid 0x%x\n", smp_processor_id(), data->currfid, data->currvid, reqfid); @@ -409,15 +385,13 @@ static int core_frequency_transition(struct powernow_k8_data *data, u32 reqfid) return 1; if (data->currfid != reqfid) { - printk(KERN_ERR PFX - "ph2: mismatch, failed fid transition, " - "curr 0x%x, req 0x%x\n", + pr_err("ph2: mismatch, failed fid transition, curr 0x%x, req 0x%x\n", data->currfid, reqfid); return 1; } if (savevid != data->currvid) { - printk(KERN_ERR PFX "ph2: vid changed, save 0x%x, curr 0x%x\n", + pr_err("ph2: vid changed, save 0x%x, curr 0x%x\n", savevid, data->currvid); return 1; } @@ -444,17 +418,14 @@ static int core_voltage_post_transition(struct powernow_k8_data *data, return 1; if (savefid != data->currfid) { - printk(KERN_ERR PFX - "ph3: bad fid change, save 0x%x, curr 0x%x\n", - savefid, data->currfid); + pr_err("ph3: bad fid change, save 0x%x, curr 0x%x\n", + savefid, data->currfid); return 1; } if (data->currvid != reqvid) { - printk(KERN_ERR PFX - "ph3: failed vid transition\n, " - "req 0x%x, curr 0x%x", - reqvid, data->currvid); + pr_err("ph3: failed vid transition\n, req 0x%x, curr 0x%x", + reqvid, data->currvid); return 1; } } @@ -481,7 +452,7 @@ static int core_voltage_post_transition(struct powernow_k8_data *data, static const struct x86_cpu_id powernow_k8_ids[] = { /* IO based frequency switching */ - { X86_VENDOR_AMD, 0xf }, + X86_MATCH_VENDOR_FAM(AMD, 0xf, NULL), {} }; MODULE_DEVICE_TABLE(x86cpu, powernow_k8_ids); @@ -498,23 +469,20 @@ static void check_supported_cpu(void *_rc) if ((eax & CPUID_XFAM) == CPUID_XFAM_K8) { if (((eax & CPUID_USE_XFAM_XMOD) != CPUID_USE_XFAM_XMOD) || ((eax & CPUID_XMOD) > CPUID_XMOD_REV_MASK)) { - printk(KERN_INFO PFX - "Processor cpuid %x not supported\n", eax); + pr_info("Processor cpuid %x not supported\n", eax); return; } eax = cpuid_eax(CPUID_GET_MAX_CAPABILITIES); if (eax < CPUID_FREQ_VOLT_CAPABILITIES) { - printk(KERN_INFO PFX - "No frequency change capabilities detected\n"); + pr_info("No frequency change capabilities detected\n"); return; } cpuid(CPUID_FREQ_VOLT_CAPABILITIES, &eax, &ebx, &ecx, &edx); if ((edx & P_STATE_TRANSITION_CAPABLE) != P_STATE_TRANSITION_CAPABLE) { - printk(KERN_INFO PFX - "Power state transitions not supported\n"); + pr_info_once("Power state transitions not supported\n"); return; } *rc = 0; @@ -529,43 +497,39 @@ static int check_pst_table(struct powernow_k8_data *data, struct pst_s *pst, for (j = 0; j < data->numps; j++) { if (pst[j].vid > LEAST_VID) { - printk(KERN_ERR FW_BUG PFX "vid %d invalid : 0x%x\n", - j, pst[j].vid); + pr_err(FW_BUG "vid %d invalid : 0x%x\n", j, + pst[j].vid); return -EINVAL; } if (pst[j].vid < data->rvo) { /* vid + rvo >= 0 */ - printk(KERN_ERR FW_BUG PFX "0 vid exceeded with pstate" - " %d\n", j); + pr_err(FW_BUG "0 vid exceeded with pstate %d\n", j); return -ENODEV; } if (pst[j].vid < maxvid + data->rvo) { /* vid + rvo >= maxvid */ - printk(KERN_ERR FW_BUG PFX "maxvid exceeded with pstate" - " %d\n", j); + pr_err(FW_BUG "maxvid exceeded with pstate %d\n", j); return -ENODEV; } if (pst[j].fid > MAX_FID) { - printk(KERN_ERR FW_BUG PFX "maxfid exceeded with pstate" - " %d\n", j); + pr_err(FW_BUG "maxfid exceeded with pstate %d\n", j); return -ENODEV; } if (j && (pst[j].fid < HI_FID_TABLE_BOTTOM)) { /* Only first fid is allowed to be in "low" range */ - printk(KERN_ERR FW_BUG PFX "two low fids - %d : " - "0x%x\n", j, pst[j].fid); + pr_err(FW_BUG "two low fids - %d : 0x%x\n", j, + pst[j].fid); return -EINVAL; } if (pst[j].fid < lastfid) lastfid = pst[j].fid; } if (lastfid & 1) { - printk(KERN_ERR FW_BUG PFX "lastfid invalid\n"); + pr_err(FW_BUG "lastfid invalid\n"); return -EINVAL; } if (lastfid > LO_FID_TABLE_TOP) - printk(KERN_INFO FW_BUG PFX - "first fid not from lo freq table\n"); + pr_info(FW_BUG "first fid not from lo freq table\n"); return 0; } @@ -582,16 +546,14 @@ static void print_basics(struct powernow_k8_data *data) for (j = 0; j < data->numps; j++) { if (data->powernow_table[j].frequency != CPUFREQ_ENTRY_INVALID) { - printk(KERN_INFO PFX - "fid 0x%x (%d MHz), vid 0x%x\n", - data->powernow_table[j].driver_data & 0xff, - data->powernow_table[j].frequency/1000, - data->powernow_table[j].driver_data >> 8); + pr_info("fid 0x%x (%d MHz), vid 0x%x\n", + data->powernow_table[j].driver_data & 0xff, + data->powernow_table[j].frequency/1000, + data->powernow_table[j].driver_data >> 8); } } if (data->batps) - printk(KERN_INFO PFX "Only %d pstates on battery\n", - data->batps); + pr_info("Only %d pstates on battery\n", data->batps); } static int fill_powernow_table(struct powernow_k8_data *data, @@ -602,33 +564,30 @@ static int fill_powernow_table(struct powernow_k8_data *data, if (data->batps) { /* use ACPI support to get full speed on mains power */ - printk(KERN_WARNING PFX - "Only %d pstates usable (use ACPI driver for full " - "range\n", data->batps); + pr_warn("Only %d pstates usable (use ACPI driver for full range\n", + data->batps); data->numps = data->batps; } for (j = 1; j < data->numps; j++) { if (pst[j-1].fid >= pst[j].fid) { - printk(KERN_ERR PFX "PST out of sequence\n"); + pr_err("PST out of sequence\n"); return -EINVAL; } } if (data->numps < 2) { - printk(KERN_ERR PFX "no p states to transition\n"); + pr_err("no p states to transition\n"); return -ENODEV; } if (check_pst_table(data, pst, maxvid)) return -EINVAL; - powernow_table = kmalloc((sizeof(struct cpufreq_frequency_table) + powernow_table = kzalloc((sizeof(*powernow_table) * (data->numps + 1)), GFP_KERNEL); - if (!powernow_table) { - printk(KERN_ERR PFX "powernow_table memory alloc failure\n"); + if (!powernow_table) return -ENOMEM; - } for (j = 0; j < data->numps; j++) { int freq; @@ -647,7 +606,7 @@ static int fill_powernow_table(struct powernow_k8_data *data, pr_debug("cfid 0x%x, cvid 0x%x\n", data->currfid, data->currvid); data->powernow_table = powernow_table; - if (cpumask_first(cpu_core_mask(data->cpu)) == data->cpu) + if (cpumask_first(topology_core_cpumask(data->cpu)) == data->cpu) print_basics(data); for (j = 0; j < data->numps; j++) @@ -681,13 +640,13 @@ static int find_psb_table(struct powernow_k8_data *data) pr_debug("table vers: 0x%x\n", psb->tableversion); if (psb->tableversion != PSB_VERSION_1_4) { - printk(KERN_ERR FW_BUG PFX "PSB table is not v1.4\n"); + pr_err(FW_BUG "PSB table is not v1.4\n"); return -ENODEV; } pr_debug("flags: 0x%x\n", psb->flags1); if (psb->flags1) { - printk(KERN_ERR FW_BUG PFX "unknown flags\n"); + pr_err(FW_BUG "unknown flags\n"); return -ENODEV; } @@ -716,7 +675,7 @@ static int find_psb_table(struct powernow_k8_data *data) cpst = 1; } if (cpst != 1) { - printk(KERN_ERR FW_BUG PFX "numpst must be 1\n"); + pr_err(FW_BUG "numpst must be 1\n"); return -ENODEV; } @@ -742,9 +701,8 @@ static int find_psb_table(struct powernow_k8_data *data) * BIOS and Kernel Developer's Guide, which is available on * www.amd.com */ - printk(KERN_ERR FW_BUG PFX "No PSB or ACPI _PSS objects\n"); - printk(KERN_ERR PFX "Make sure that your BIOS is up to date" - " and Cool'N'Quiet support is enabled in BIOS setup\n"); + pr_err(FW_BUG "No PSB or ACPI _PSS objects\n"); + pr_err("Make sure that your BIOS is up to date and Cool'N'Quiet support is enabled in BIOS setup\n"); return -ENODEV; } @@ -793,12 +751,10 @@ static int powernow_k8_cpu_init_acpi(struct powernow_k8_data *data) } /* fill in data->powernow_table */ - powernow_table = kmalloc((sizeof(struct cpufreq_frequency_table) + powernow_table = kzalloc((sizeof(*powernow_table) * (data->acpi_data.state_count + 1)), GFP_KERNEL); - if (!powernow_table) { - pr_debug("powernow_table memory alloc failure\n"); + if (!powernow_table) goto err_out; - } /* fill in data */ data->numps = data->acpi_data.state_count; @@ -810,18 +766,16 @@ static int powernow_k8_cpu_init_acpi(struct powernow_k8_data *data) powernow_table[data->acpi_data.state_count].frequency = CPUFREQ_TABLE_END; - powernow_table[data->acpi_data.state_count].driver_data = 0; data->powernow_table = powernow_table; - if (cpumask_first(cpu_core_mask(data->cpu)) == data->cpu) + if (cpumask_first(topology_core_cpumask(data->cpu)) == data->cpu) print_basics(data); /* notify BIOS that we exist */ acpi_processor_notify_smm(THIS_MODULE); if (!zalloc_cpumask_var(&data->acpi_data.shared_cpu_map, GFP_KERNEL)) { - printk(KERN_ERR PFX - "unable to alloc powernow_k8_data cpumask\n"); + pr_err("unable to alloc powernow_k8_data cpumask\n"); ret_val = -ENOMEM; goto err_out_mem; } @@ -832,7 +786,7 @@ err_out_mem: kfree(powernow_table); err_out: - acpi_processor_unregister_performance(&data->acpi_data, data->cpu); + acpi_processor_unregister_performance(data->cpu); /* data->acpi_data.state_count informs us at ->exit() * whether ACPI was used */ @@ -886,9 +840,8 @@ static int fill_powernow_table_fidvid(struct powernow_k8_data *data, } if (freq != (data->acpi_data.states[i].core_frequency * 1000)) { - printk(KERN_INFO PFX "invalid freq entries " - "%u kHz vs. %u kHz\n", freq, - (unsigned int) + pr_info("invalid freq entries %u kHz vs. %u kHz\n", + freq, (unsigned int) (data->acpi_data.states[i].core_frequency * 1000)); invalidate_entry(powernow_table, i); @@ -901,8 +854,7 @@ static int fill_powernow_table_fidvid(struct powernow_k8_data *data, static void powernow_k8_cpu_exit_acpi(struct powernow_k8_data *data) { if (data->acpi_data.state_count) - acpi_processor_unregister_performance(&data->acpi_data, - data->cpu); + acpi_processor_unregister_performance(data->cpu); free_cpumask_var(data->acpi_data.shared_cpu_map); } @@ -917,7 +869,7 @@ static int get_transition_latency(struct powernow_k8_data *data) max_latency = cur_latency; } if (max_latency == 0) { - pr_err(FW_WARN PFX "Invalid zero transition latency\n"); + pr_err(FW_WARN "Invalid zero transition latency\n"); max_latency = 1; } /* value in usecs, needs to be in nanoseconds */ @@ -926,9 +878,9 @@ static int get_transition_latency(struct powernow_k8_data *data) /* Take a frequency, and issue the fid/vid transition command */ static int transition_frequency_fidvid(struct powernow_k8_data *data, - unsigned int index) + unsigned int index, + struct cpufreq_policy *policy) { - struct cpufreq_policy *policy; u32 fid = 0; u32 vid = 0; int res; @@ -960,37 +912,26 @@ static int transition_frequency_fidvid(struct powernow_k8_data *data, freqs.old = find_khz_freq_from_fid(data->currfid); freqs.new = find_khz_freq_from_fid(fid); - policy = cpufreq_cpu_get(smp_processor_id()); - cpufreq_cpu_put(policy); - - cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE); - + cpufreq_freq_transition_begin(policy, &freqs); res = transition_fid_vid(data, fid, vid); - if (res) - freqs.new = freqs.old; - else - freqs.new = find_khz_freq_from_fid(data->currfid); + cpufreq_freq_transition_end(policy, &freqs, res); - cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); return res; } struct powernowk8_target_arg { struct cpufreq_policy *pol; - unsigned targfreq; - unsigned relation; + unsigned newstate; }; static long powernowk8_target_fn(void *arg) { struct powernowk8_target_arg *pta = arg; struct cpufreq_policy *pol = pta->pol; - unsigned targfreq = pta->targfreq; - unsigned relation = pta->relation; + unsigned newstate = pta->newstate; struct powernow_k8_data *data = per_cpu(powernow_data, pol->cpu); u32 checkfid; u32 checkvid; - unsigned int newstate; int ret; if (!data) @@ -1000,39 +941,35 @@ static long powernowk8_target_fn(void *arg) checkvid = data->currvid; if (pending_bit_stuck()) { - printk(KERN_ERR PFX "failing targ, change pending bit set\n"); + pr_err("failing targ, change pending bit set\n"); return -EIO; } - pr_debug("targ: cpu %d, %d kHz, min %d, max %d, relation %d\n", - pol->cpu, targfreq, pol->min, pol->max, relation); + pr_debug("targ: cpu %d, %d kHz, min %d, max %d\n", + pol->cpu, data->powernow_table[newstate].frequency, pol->min, + pol->max); if (query_current_values_with_pending_wait(data)) return -EIO; pr_debug("targ: curr fid 0x%x, vid 0x%x\n", - data->currfid, data->currvid); + data->currfid, data->currvid); if ((checkvid != data->currvid) || (checkfid != data->currfid)) { - pr_info(PFX - "error - out of sync, fix 0x%x 0x%x, vid 0x%x 0x%x\n", + pr_info("error - out of sync, fix 0x%x 0x%x, vid 0x%x 0x%x\n", checkfid, data->currfid, checkvid, data->currvid); } - if (cpufreq_frequency_table_target(pol, data->powernow_table, - targfreq, relation, &newstate)) - return -EIO; - mutex_lock(&fidvid_mutex); powernow_k8_acpi_pst_values(data, newstate); - ret = transition_frequency_fidvid(data, newstate); + ret = transition_frequency_fidvid(data, newstate, pol); if (ret) { - printk(KERN_ERR PFX "transition frequency failed\n"); + pr_err("transition frequency failed\n"); mutex_unlock(&fidvid_mutex); return 1; } @@ -1044,37 +981,24 @@ static long powernowk8_target_fn(void *arg) } /* Driver entry point to switch to the target frequency */ -static int powernowk8_target(struct cpufreq_policy *pol, - unsigned targfreq, unsigned relation) +static int powernowk8_target(struct cpufreq_policy *pol, unsigned index) { - struct powernowk8_target_arg pta = { .pol = pol, .targfreq = targfreq, - .relation = relation }; + struct powernowk8_target_arg pta = { .pol = pol, .newstate = index }; return work_on_cpu(pol->cpu, powernowk8_target_fn, &pta); } -/* Driver entry point to verify the policy and range of frequencies */ -static int powernowk8_verify(struct cpufreq_policy *pol) -{ - struct powernow_k8_data *data = per_cpu(powernow_data, pol->cpu); - - if (!data) - return -EINVAL; - - return cpufreq_frequency_table_verify(pol, data->powernow_table); -} - struct init_on_cpu { struct powernow_k8_data *data; int rc; }; -static void __cpuinit powernowk8_cpu_init_on_cpu(void *_init_on_cpu) +static void powernowk8_cpu_init_on_cpu(void *_init_on_cpu) { struct init_on_cpu *init_on_cpu = _init_on_cpu; if (pending_bit_stuck()) { - printk(KERN_ERR PFX "failing init, change pending bit set\n"); + pr_err("failing init, change pending bit set\n"); init_on_cpu->rc = -ENODEV; return; } @@ -1089,28 +1013,25 @@ static void __cpuinit powernowk8_cpu_init_on_cpu(void *_init_on_cpu) init_on_cpu->rc = 0; } -static const char missing_pss_msg[] = - KERN_ERR - FW_BUG PFX "No compatible ACPI _PSS objects found.\n" - FW_BUG PFX "First, make sure Cool'N'Quiet is enabled in the BIOS.\n" - FW_BUG PFX "If that doesn't help, try upgrading your BIOS.\n"; +#define MISSING_PSS_MSG \ + FW_BUG "No compatible ACPI _PSS objects found.\n" \ + FW_BUG "First, make sure Cool'N'Quiet is enabled in the BIOS.\n" \ + FW_BUG "If that doesn't help, try upgrading your BIOS.\n" /* per CPU init entry point to the driver */ -static int __cpuinit powernowk8_cpu_init(struct cpufreq_policy *pol) +static int powernowk8_cpu_init(struct cpufreq_policy *pol) { struct powernow_k8_data *data; struct init_on_cpu init_on_cpu; - int rc; + int rc, cpu; smp_call_function_single(pol->cpu, check_supported_cpu, &rc, 1); if (rc) return -ENODEV; - data = kzalloc(sizeof(struct powernow_k8_data), GFP_KERNEL); - if (!data) { - printk(KERN_ERR PFX "unable to alloc powernow_k8_data"); + data = kzalloc(sizeof(*data), GFP_KERNEL); + if (!data) return -ENOMEM; - } data->cpu = pol->cpu; @@ -1120,13 +1041,11 @@ static int __cpuinit powernowk8_cpu_init(struct cpufreq_policy *pol) * an UP version, and is deprecated by AMD. */ if (num_online_cpus() != 1) { - printk_once(missing_pss_msg); + pr_err_once(MISSING_PSS_MSG); goto err_out; } if (pol->cpu != 0) { - printk(KERN_ERR FW_BUG PFX "No ACPI _PSS objects for " - "CPU other than CPU0. Complain to your BIOS " - "vendor.\n"); + pr_err(FW_BUG "No ACPI _PSS objects for CPU other than CPU0. Complain to your BIOS vendor.\n"); goto err_out; } rc = find_psb_table(data); @@ -1149,27 +1068,16 @@ static int __cpuinit powernowk8_cpu_init(struct cpufreq_policy *pol) if (rc != 0) goto err_out_exit_acpi; - cpumask_copy(pol->cpus, cpu_core_mask(pol->cpu)); + cpumask_copy(pol->cpus, topology_core_cpumask(pol->cpu)); data->available_cores = pol->cpus; - - pol->cur = find_khz_freq_from_fid(data->currfid); - pr_debug("policy current frequency %d kHz\n", pol->cur); - - /* min/max the cpu is capable of */ - if (cpufreq_frequency_table_cpuinfo(pol, data->powernow_table)) { - printk(KERN_ERR FW_BUG PFX "invalid powernow_table\n"); - powernow_k8_cpu_exit_acpi(data); - kfree(data->powernow_table); - kfree(data); - return -EINVAL; - } - - cpufreq_frequency_table_get_attr(data->powernow_table, pol->cpu); + pol->freq_table = data->powernow_table; pr_debug("cpu_init done, current fid 0x%x, vid 0x%x\n", - data->currfid, data->currvid); + data->currfid, data->currvid); - per_cpu(powernow_data, pol->cpu) = data; + /* Point all the CPUs in this policy to the same data */ + for_each_cpu(cpu, pol->cpus) + per_cpu(powernow_data, cpu) = data; return 0; @@ -1181,22 +1089,21 @@ err_out: return -ENODEV; } -static int powernowk8_cpu_exit(struct cpufreq_policy *pol) +static void powernowk8_cpu_exit(struct cpufreq_policy *pol) { struct powernow_k8_data *data = per_cpu(powernow_data, pol->cpu); + int cpu; if (!data) - return -EINVAL; + return; powernow_k8_cpu_exit_acpi(data); - cpufreq_frequency_table_put_attr(pol->cpu); - kfree(data->powernow_table); kfree(data); - per_cpu(powernow_data, pol->cpu) = NULL; - - return 0; + /* pol->cpus will be empty here, use related_cpus instead. */ + for_each_cpu(cpu, pol->related_cpus) + per_cpu(powernow_data, cpu) = NULL; } static void query_values_on_cpu(void *_err) @@ -1227,56 +1134,51 @@ out: return khz; } -static struct freq_attr *powernow_k8_attr[] = { - &cpufreq_freq_attr_scaling_available_freqs, - NULL, -}; - static struct cpufreq_driver cpufreq_amd64_driver = { - .verify = powernowk8_verify, - .target = powernowk8_target, + .flags = CPUFREQ_ASYNC_NOTIFICATION, + .verify = cpufreq_generic_frequency_table_verify, + .target_index = powernowk8_target, .bios_limit = acpi_processor_get_bios_limit, .init = powernowk8_cpu_init, .exit = powernowk8_cpu_exit, .get = powernowk8_get, .name = "powernow-k8", - .owner = THIS_MODULE, - .attr = powernow_k8_attr, }; static void __request_acpi_cpufreq(void) { - const char *cur_drv, *drv = "acpi-cpufreq"; + const char drv[] = "acpi-cpufreq"; + const char *cur_drv; cur_drv = cpufreq_get_current_driver(); if (!cur_drv) goto request; if (strncmp(cur_drv, drv, min_t(size_t, strlen(cur_drv), strlen(drv)))) - pr_warn(PFX "WTF driver: %s\n", cur_drv); + pr_warn("WTF driver: %s\n", cur_drv); return; request: - pr_warn(PFX "This CPU is not supported anymore, using acpi-cpufreq instead.\n"); + pr_warn("This CPU is not supported anymore, using acpi-cpufreq instead.\n"); request_module(drv); } /* driver entry point for init */ -static int __cpuinit powernowk8_init(void) +static int powernowk8_init(void) { unsigned int i, supported_cpus = 0; int ret; - if (static_cpu_has(X86_FEATURE_HW_PSTATE)) { - __request_acpi_cpufreq(); + if (!x86_match_cpu(powernow_k8_ids)) return -ENODEV; - } - if (!x86_match_cpu(powernow_k8_ids)) + if (boot_cpu_has(X86_FEATURE_HW_PSTATE)) { + __request_acpi_cpufreq(); return -ENODEV; + } - get_online_cpus(); + cpus_read_lock(); for_each_online_cpu(i) { smp_call_function_single(i, check_supported_cpu, &ret, 1); if (!ret) @@ -1284,16 +1186,16 @@ static int __cpuinit powernowk8_init(void) } if (supported_cpus != num_online_cpus()) { - put_online_cpus(); + cpus_read_unlock(); return -ENODEV; } - put_online_cpus(); + cpus_read_unlock(); ret = cpufreq_register_driver(&cpufreq_amd64_driver); if (ret) return ret; - pr_info(PFX "Found %d %s (%d cpu cores) (" VERSION ")\n", + pr_info("Found %d %s (%d cpu cores) (" VERSION ")\n", num_online_nodes(), boot_cpu_data.x86_model_id, supported_cpus); return ret; @@ -1307,8 +1209,8 @@ static void __exit powernowk8_exit(void) cpufreq_unregister_driver(&cpufreq_amd64_driver); } -MODULE_AUTHOR("Paul Devriendt <paul.devriendt@amd.com> and " - "Mark Langsdorf <mark.langsdorf@amd.com>"); +MODULE_AUTHOR("Paul Devriendt <paul.devriendt@amd.com>"); +MODULE_AUTHOR("Mark Langsdorf <mark.langsdorf@amd.com>"); MODULE_DESCRIPTION("AMD Athlon 64 and Opteron processor frequency driver."); MODULE_LICENSE("GPL"); diff --git a/drivers/cpufreq/powernow-k8.h b/drivers/cpufreq/powernow-k8.h index 79329d4d5abe..83331ceb6bc3 100644 --- a/drivers/cpufreq/powernow-k8.h +++ b/drivers/cpufreq/powernow-k8.h @@ -1,8 +1,6 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * (c) 2003-2006 Advanced Micro Devices, Inc. - * Your use of this code is subject to the terms and conditions of the - * GNU general public license version 2. See "COPYING" or - * http://www.gnu.org/licenses/gpl.html */ struct powernow_k8_data { @@ -19,7 +17,7 @@ struct powernow_k8_data { u32 vidmvs; /* usable value calculated from mvs */ u32 vstable; /* voltage stabilization time, units 20 us */ u32 plllock; /* pll lock time, units 1 us */ - u32 exttype; /* extended interface = 1 */ + u32 exttype; /* extended interface = 1 */ /* keep track of the current fid / vid or pstate */ u32 currvid; diff --git a/drivers/cpufreq/powernv-cpufreq.c b/drivers/cpufreq/powernv-cpufreq.c new file mode 100644 index 000000000000..7d9a5f656de8 --- /dev/null +++ b/drivers/cpufreq/powernv-cpufreq.c @@ -0,0 +1,1160 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * POWERNV cpufreq driver for the IBM POWER processors + * + * (C) Copyright IBM 2014 + * + * Author: Vaidyanathan Srinivasan <svaidy at linux.vnet.ibm.com> + */ + +#define pr_fmt(fmt) "powernv-cpufreq: " fmt + +#include <linux/kernel.h> +#include <linux/sysfs.h> +#include <linux/cpumask.h> +#include <linux/module.h> +#include <linux/cpufreq.h> +#include <linux/smp.h> +#include <linux/of.h> +#include <linux/reboot.h> +#include <linux/slab.h> +#include <linux/string_choices.h> +#include <linux/cpu.h> +#include <linux/hashtable.h> + +#include <asm/cputhreads.h> +#include <asm/firmware.h> +#include <asm/reg.h> +#include <asm/smp.h> /* Required for cpu_sibling_mask() in UP configs */ +#include <asm/opal.h> +#include <linux/timer.h> + +#define CREATE_TRACE_POINTS +#include "powernv-trace.h" + +#define POWERNV_MAX_PSTATES_ORDER 8 +#define POWERNV_MAX_PSTATES (1UL << (POWERNV_MAX_PSTATES_ORDER)) +#define PMSR_PSAFE_ENABLE (1UL << 30) +#define PMSR_SPR_EM_DISABLE (1UL << 31) +#define MAX_PSTATE_SHIFT 32 +#define LPSTATE_SHIFT 48 +#define GPSTATE_SHIFT 56 +#define MAX_NR_CHIPS 32 + +#define MAX_RAMP_DOWN_TIME 5120 +/* + * On an idle system we want the global pstate to ramp-down from max value to + * min over a span of ~5 secs. Also we want it to initially ramp-down slowly and + * then ramp-down rapidly later on. + * + * This gives a percentage rampdown for time elapsed in milliseconds. + * ramp_down_percentage = ((ms * ms) >> 18) + * ~= 3.8 * (sec * sec) + * + * At 0 ms ramp_down_percent = 0 + * At 5120 ms ramp_down_percent = 100 + */ +#define ramp_down_percent(time) ((time * time) >> 18) + +/* Interval after which the timer is queued to bring down global pstate */ +#define GPSTATE_TIMER_INTERVAL 2000 + +/** + * struct global_pstate_info - Per policy data structure to maintain history of + * global pstates + * @highest_lpstate_idx: The local pstate index from which we are + * ramping down + * @elapsed_time: Time in ms spent in ramping down from + * highest_lpstate_idx + * @last_sampled_time: Time from boot in ms when global pstates were + * last set + * @last_lpstate_idx: Last set value of local pstate and global + * @last_gpstate_idx: pstate in terms of cpufreq table index + * @timer: Is used for ramping down if cpu goes idle for + * a long time with global pstate held high + * @gpstate_lock: A spinlock to maintain synchronization between + * routines called by the timer handler and + * governer's target_index calls + * @policy: Associated CPUFreq policy + */ +struct global_pstate_info { + int highest_lpstate_idx; + unsigned int elapsed_time; + unsigned int last_sampled_time; + int last_lpstate_idx; + int last_gpstate_idx; + spinlock_t gpstate_lock; + struct timer_list timer; + struct cpufreq_policy *policy; +}; + +static struct cpufreq_frequency_table powernv_freqs[POWERNV_MAX_PSTATES+1]; + +static DEFINE_HASHTABLE(pstate_revmap, POWERNV_MAX_PSTATES_ORDER); +/** + * struct pstate_idx_revmap_data: Entry in the hashmap pstate_revmap + * indexed by a function of pstate id. + * + * @pstate_id: pstate id for this entry. + * + * @cpufreq_table_idx: Index into the powernv_freqs + * cpufreq_frequency_table for frequency + * corresponding to pstate_id. + * + * @hentry: hlist_node that hooks this entry into the pstate_revmap + * hashtable + */ +struct pstate_idx_revmap_data { + u8 pstate_id; + unsigned int cpufreq_table_idx; + struct hlist_node hentry; +}; + +static bool rebooting, throttled, occ_reset; + +static const char * const throttle_reason[] = { + "No throttling", + "Power Cap", + "Processor Over Temperature", + "Power Supply Failure", + "Over Current", + "OCC Reset" +}; + +enum throttle_reason_type { + NO_THROTTLE = 0, + POWERCAP, + CPU_OVERTEMP, + POWER_SUPPLY_FAILURE, + OVERCURRENT, + OCC_RESET_THROTTLE, + OCC_MAX_REASON +}; + +static struct chip { + unsigned int id; + bool throttled; + bool restore; + u8 throttle_reason; + cpumask_t mask; + struct work_struct throttle; + int throttle_turbo; + int throttle_sub_turbo; + int reason[OCC_MAX_REASON]; +} *chips; + +static int nr_chips; +static DEFINE_PER_CPU(struct chip *, chip_info); + +/* + * Note: + * The set of pstates consists of contiguous integers. + * powernv_pstate_info stores the index of the frequency table for + * max, min and nominal frequencies. It also stores number of + * available frequencies. + * + * powernv_pstate_info.nominal indicates the index to the highest + * non-turbo frequency. + */ +static struct powernv_pstate_info { + unsigned int min; + unsigned int max; + unsigned int nominal; + unsigned int nr_pstates; + bool wof_enabled; +} powernv_pstate_info; + +static inline u8 extract_pstate(u64 pmsr_val, unsigned int shift) +{ + return ((pmsr_val >> shift) & 0xFF); +} + +#define extract_local_pstate(x) extract_pstate(x, LPSTATE_SHIFT) +#define extract_global_pstate(x) extract_pstate(x, GPSTATE_SHIFT) +#define extract_max_pstate(x) extract_pstate(x, MAX_PSTATE_SHIFT) + +/* Use following functions for conversions between pstate_id and index */ + +/* + * idx_to_pstate : Returns the pstate id corresponding to the + * frequency in the cpufreq frequency table + * powernv_freqs indexed by @i. + * + * If @i is out of bound, this will return the pstate + * corresponding to the nominal frequency. + */ +static inline u8 idx_to_pstate(unsigned int i) +{ + if (unlikely(i >= powernv_pstate_info.nr_pstates)) { + pr_warn_once("idx_to_pstate: index %u is out of bound\n", i); + return powernv_freqs[powernv_pstate_info.nominal].driver_data; + } + + return powernv_freqs[i].driver_data; +} + +/* + * pstate_to_idx : Returns the index in the cpufreq frequencytable + * powernv_freqs for the frequency whose corresponding + * pstate id is @pstate. + * + * If no frequency corresponding to @pstate is found, + * this will return the index of the nominal + * frequency. + */ +static unsigned int pstate_to_idx(u8 pstate) +{ + unsigned int key = pstate % POWERNV_MAX_PSTATES; + struct pstate_idx_revmap_data *revmap_data; + + hash_for_each_possible(pstate_revmap, revmap_data, hentry, key) { + if (revmap_data->pstate_id == pstate) + return revmap_data->cpufreq_table_idx; + } + + pr_warn_once("pstate_to_idx: pstate 0x%x not found\n", pstate); + return powernv_pstate_info.nominal; +} + +static inline void reset_gpstates(struct cpufreq_policy *policy) +{ + struct global_pstate_info *gpstates = policy->driver_data; + + gpstates->highest_lpstate_idx = 0; + gpstates->elapsed_time = 0; + gpstates->last_sampled_time = 0; + gpstates->last_lpstate_idx = 0; + gpstates->last_gpstate_idx = 0; +} + +/* + * Initialize the freq table based on data obtained + * from the firmware passed via device-tree + */ +static int init_powernv_pstates(void) +{ + struct device_node *power_mgt; + int i, nr_pstates = 0; + const __be32 *pstate_ids, *pstate_freqs; + u32 len_ids, len_freqs; + u32 pstate_min, pstate_max, pstate_nominal; + u32 pstate_turbo, pstate_ultra_turbo; + int rc = -ENODEV; + + power_mgt = of_find_node_by_path("/ibm,opal/power-mgt"); + if (!power_mgt) { + pr_warn("power-mgt node not found\n"); + return -ENODEV; + } + + if (of_property_read_u32(power_mgt, "ibm,pstate-min", &pstate_min)) { + pr_warn("ibm,pstate-min node not found\n"); + goto out; + } + + if (of_property_read_u32(power_mgt, "ibm,pstate-max", &pstate_max)) { + pr_warn("ibm,pstate-max node not found\n"); + goto out; + } + + if (of_property_read_u32(power_mgt, "ibm,pstate-nominal", + &pstate_nominal)) { + pr_warn("ibm,pstate-nominal not found\n"); + goto out; + } + + if (of_property_read_u32(power_mgt, "ibm,pstate-ultra-turbo", + &pstate_ultra_turbo)) { + powernv_pstate_info.wof_enabled = false; + goto next; + } + + if (of_property_read_u32(power_mgt, "ibm,pstate-turbo", + &pstate_turbo)) { + powernv_pstate_info.wof_enabled = false; + goto next; + } + + if (pstate_turbo == pstate_ultra_turbo) + powernv_pstate_info.wof_enabled = false; + else + powernv_pstate_info.wof_enabled = true; + +next: + pr_info("cpufreq pstate min 0x%x nominal 0x%x max 0x%x\n", pstate_min, + pstate_nominal, pstate_max); + pr_info("Workload Optimized Frequency is %s in the platform\n", + str_enabled_disabled(powernv_pstate_info.wof_enabled)); + + pstate_ids = of_get_property(power_mgt, "ibm,pstate-ids", &len_ids); + if (!pstate_ids) { + pr_warn("ibm,pstate-ids not found\n"); + goto out; + } + + pstate_freqs = of_get_property(power_mgt, "ibm,pstate-frequencies-mhz", + &len_freqs); + if (!pstate_freqs) { + pr_warn("ibm,pstate-frequencies-mhz not found\n"); + goto out; + } + + if (len_ids != len_freqs) { + pr_warn("Entries in ibm,pstate-ids and " + "ibm,pstate-frequencies-mhz does not match\n"); + } + + nr_pstates = min(len_ids, len_freqs) / sizeof(u32); + if (!nr_pstates) { + pr_warn("No PStates found\n"); + goto out; + } + + powernv_pstate_info.nr_pstates = nr_pstates; + pr_debug("NR PStates %d\n", nr_pstates); + + for (i = 0; i < nr_pstates; i++) { + u32 id = be32_to_cpu(pstate_ids[i]); + u32 freq = be32_to_cpu(pstate_freqs[i]); + struct pstate_idx_revmap_data *revmap_data; + unsigned int key; + + pr_debug("PState id %d freq %d MHz\n", id, freq); + powernv_freqs[i].frequency = freq * 1000; /* kHz */ + powernv_freqs[i].driver_data = id & 0xFF; + + revmap_data = kmalloc(sizeof(*revmap_data), GFP_KERNEL); + if (!revmap_data) { + rc = -ENOMEM; + goto out; + } + + revmap_data->pstate_id = id & 0xFF; + revmap_data->cpufreq_table_idx = i; + key = (revmap_data->pstate_id) % POWERNV_MAX_PSTATES; + hash_add(pstate_revmap, &revmap_data->hentry, key); + + if (id == pstate_max) + powernv_pstate_info.max = i; + if (id == pstate_nominal) + powernv_pstate_info.nominal = i; + if (id == pstate_min) + powernv_pstate_info.min = i; + + if (powernv_pstate_info.wof_enabled && id == pstate_turbo) { + int j; + + for (j = i - 1; j >= (int)powernv_pstate_info.max; j--) + powernv_freqs[j].flags = CPUFREQ_BOOST_FREQ; + } + } + + /* End of list marker entry */ + powernv_freqs[i].frequency = CPUFREQ_TABLE_END; + + of_node_put(power_mgt); + return 0; +out: + of_node_put(power_mgt); + return rc; +} + +/* Returns the CPU frequency corresponding to the pstate_id. */ +static unsigned int pstate_id_to_freq(u8 pstate_id) +{ + int i; + + i = pstate_to_idx(pstate_id); + if (i >= powernv_pstate_info.nr_pstates || i < 0) { + pr_warn("PState id 0x%x outside of PState table, reporting nominal id 0x%x instead\n", + pstate_id, idx_to_pstate(powernv_pstate_info.nominal)); + i = powernv_pstate_info.nominal; + } + + return powernv_freqs[i].frequency; +} + +/* + * cpuinfo_nominal_freq_show - Show the nominal CPU frequency as indicated by + * the firmware + */ +static ssize_t cpuinfo_nominal_freq_show(struct cpufreq_policy *policy, + char *buf) +{ + return sprintf(buf, "%u\n", + powernv_freqs[powernv_pstate_info.nominal].frequency); +} + +static struct freq_attr cpufreq_freq_attr_cpuinfo_nominal_freq = + __ATTR_RO(cpuinfo_nominal_freq); + +static struct freq_attr *powernv_cpu_freq_attr[] = { + &cpufreq_freq_attr_cpuinfo_nominal_freq, + NULL, +}; + +#define throttle_attr(name, member) \ +static ssize_t name##_show(struct cpufreq_policy *policy, char *buf) \ +{ \ + struct chip *chip = per_cpu(chip_info, policy->cpu); \ + \ + return sprintf(buf, "%u\n", chip->member); \ +} \ + \ +static struct freq_attr throttle_attr_##name = __ATTR_RO(name) \ + +throttle_attr(unthrottle, reason[NO_THROTTLE]); +throttle_attr(powercap, reason[POWERCAP]); +throttle_attr(overtemp, reason[CPU_OVERTEMP]); +throttle_attr(supply_fault, reason[POWER_SUPPLY_FAILURE]); +throttle_attr(overcurrent, reason[OVERCURRENT]); +throttle_attr(occ_reset, reason[OCC_RESET_THROTTLE]); +throttle_attr(turbo_stat, throttle_turbo); +throttle_attr(sub_turbo_stat, throttle_sub_turbo); + +static struct attribute *throttle_attrs[] = { + &throttle_attr_unthrottle.attr, + &throttle_attr_powercap.attr, + &throttle_attr_overtemp.attr, + &throttle_attr_supply_fault.attr, + &throttle_attr_overcurrent.attr, + &throttle_attr_occ_reset.attr, + &throttle_attr_turbo_stat.attr, + &throttle_attr_sub_turbo_stat.attr, + NULL, +}; + +static const struct attribute_group throttle_attr_grp = { + .name = "throttle_stats", + .attrs = throttle_attrs, +}; + +/* Helper routines */ + +/* Access helpers to power mgt SPR */ + +static inline unsigned long get_pmspr(unsigned long sprn) +{ + switch (sprn) { + case SPRN_PMCR: + return mfspr(SPRN_PMCR); + + case SPRN_PMICR: + return mfspr(SPRN_PMICR); + + case SPRN_PMSR: + return mfspr(SPRN_PMSR); + } + BUG(); +} + +static inline void set_pmspr(unsigned long sprn, unsigned long val) +{ + switch (sprn) { + case SPRN_PMCR: + mtspr(SPRN_PMCR, val); + return; + + case SPRN_PMICR: + mtspr(SPRN_PMICR, val); + return; + } + BUG(); +} + +/* + * Use objects of this type to query/update + * pstates on a remote CPU via smp_call_function. + */ +struct powernv_smp_call_data { + unsigned int freq; + u8 pstate_id; + u8 gpstate_id; +}; + +/* + * powernv_read_cpu_freq: Reads the current frequency on this CPU. + * + * Called via smp_call_function. + * + * Note: The caller of the smp_call_function should pass an argument of + * the type 'struct powernv_smp_call_data *' along with this function. + * + * The current frequency on this CPU will be returned via + * ((struct powernv_smp_call_data *)arg)->freq; + */ +static void powernv_read_cpu_freq(void *arg) +{ + unsigned long pmspr_val; + struct powernv_smp_call_data *freq_data = arg; + + pmspr_val = get_pmspr(SPRN_PMSR); + freq_data->pstate_id = extract_local_pstate(pmspr_val); + freq_data->freq = pstate_id_to_freq(freq_data->pstate_id); + + pr_debug("cpu %d pmsr %016lX pstate_id 0x%x frequency %d kHz\n", + raw_smp_processor_id(), pmspr_val, freq_data->pstate_id, + freq_data->freq); +} + +/* + * powernv_cpufreq_get: Returns the CPU frequency as reported by the + * firmware for CPU 'cpu'. This value is reported through the sysfs + * file cpuinfo_cur_freq. + */ +static unsigned int powernv_cpufreq_get(unsigned int cpu) +{ + struct powernv_smp_call_data freq_data; + + smp_call_function_any(cpu_sibling_mask(cpu), powernv_read_cpu_freq, + &freq_data, 1); + + return freq_data.freq; +} + +/* + * set_pstate: Sets the pstate on this CPU. + * + * This is called via an smp_call_function. + * + * The caller must ensure that freq_data is of the type + * (struct powernv_smp_call_data *) and the pstate_id which needs to be set + * on this CPU should be present in freq_data->pstate_id. + */ +static void set_pstate(void *data) +{ + unsigned long val; + struct powernv_smp_call_data *freq_data = data; + unsigned long pstate_ul = freq_data->pstate_id; + unsigned long gpstate_ul = freq_data->gpstate_id; + + val = get_pmspr(SPRN_PMCR); + val = val & 0x0000FFFFFFFFFFFFULL; + + pstate_ul = pstate_ul & 0xFF; + gpstate_ul = gpstate_ul & 0xFF; + + /* Set both global(bits 56..63) and local(bits 48..55) PStates */ + val = val | (gpstate_ul << 56) | (pstate_ul << 48); + + pr_debug("Setting cpu %d pmcr to %016lX\n", + raw_smp_processor_id(), val); + set_pmspr(SPRN_PMCR, val); +} + +/* + * get_nominal_index: Returns the index corresponding to the nominal + * pstate in the cpufreq table + */ +static inline unsigned int get_nominal_index(void) +{ + return powernv_pstate_info.nominal; +} + +static void powernv_cpufreq_throttle_check(void *data) +{ + struct chip *chip; + unsigned int cpu = smp_processor_id(); + unsigned long pmsr; + u8 pmsr_pmax; + unsigned int pmsr_pmax_idx; + + pmsr = get_pmspr(SPRN_PMSR); + chip = this_cpu_read(chip_info); + + /* Check for Pmax Capping */ + pmsr_pmax = extract_max_pstate(pmsr); + pmsr_pmax_idx = pstate_to_idx(pmsr_pmax); + if (pmsr_pmax_idx != powernv_pstate_info.max) { + if (chip->throttled) + goto next; + chip->throttled = true; + if (pmsr_pmax_idx > powernv_pstate_info.nominal) { + pr_warn_once("CPU %d on Chip %u has Pmax(0x%x) reduced below that of nominal frequency(0x%x)\n", + cpu, chip->id, pmsr_pmax, + idx_to_pstate(powernv_pstate_info.nominal)); + chip->throttle_sub_turbo++; + } else { + chip->throttle_turbo++; + } + trace_powernv_throttle(chip->id, + throttle_reason[chip->throttle_reason], + pmsr_pmax); + } else if (chip->throttled) { + chip->throttled = false; + trace_powernv_throttle(chip->id, + throttle_reason[chip->throttle_reason], + pmsr_pmax); + } + + /* Check if Psafe_mode_active is set in PMSR. */ +next: + if (pmsr & PMSR_PSAFE_ENABLE) { + throttled = true; + pr_info("Pstate set to safe frequency\n"); + } + + /* Check if SPR_EM_DISABLE is set in PMSR */ + if (pmsr & PMSR_SPR_EM_DISABLE) { + throttled = true; + pr_info("Frequency Control disabled from OS\n"); + } + + if (throttled) { + pr_info("PMSR = %16lx\n", pmsr); + pr_warn("CPU Frequency could be throttled\n"); + } +} + +/** + * calc_global_pstate - Calculate global pstate + * @elapsed_time: Elapsed time in milliseconds + * @local_pstate_idx: New local pstate + * @highest_lpstate_idx: pstate from which its ramping down + * + * Finds the appropriate global pstate based on the pstate from which its + * ramping down and the time elapsed in ramping down. It follows a quadratic + * equation which ensures that it reaches ramping down to pmin in 5sec. + */ +static inline int calc_global_pstate(unsigned int elapsed_time, + int highest_lpstate_idx, + int local_pstate_idx) +{ + int index_diff; + + /* + * Using ramp_down_percent we get the percentage of rampdown + * that we are expecting to be dropping. Difference between + * highest_lpstate_idx and powernv_pstate_info.min will give a absolute + * number of how many pstates we will drop eventually by the end of + * 5 seconds, then just scale it get the number pstates to be dropped. + */ + index_diff = ((int)ramp_down_percent(elapsed_time) * + (powernv_pstate_info.min - highest_lpstate_idx)) / 100; + + /* Ensure that global pstate is >= to local pstate */ + if (highest_lpstate_idx + index_diff >= local_pstate_idx) + return local_pstate_idx; + else + return highest_lpstate_idx + index_diff; +} + +static inline void queue_gpstate_timer(struct global_pstate_info *gpstates) +{ + unsigned int timer_interval; + + /* + * Setting up timer to fire after GPSTATE_TIMER_INTERVAL ms, But + * if it exceeds MAX_RAMP_DOWN_TIME ms for ramp down time. + * Set timer such that it fires exactly at MAX_RAMP_DOWN_TIME + * seconds of ramp down time. + */ + if ((gpstates->elapsed_time + GPSTATE_TIMER_INTERVAL) + > MAX_RAMP_DOWN_TIME) + timer_interval = MAX_RAMP_DOWN_TIME - gpstates->elapsed_time; + else + timer_interval = GPSTATE_TIMER_INTERVAL; + + mod_timer(&gpstates->timer, jiffies + msecs_to_jiffies(timer_interval)); +} + +/** + * gpstate_timer_handler + * + * @t: Timer context used to fetch global pstate info struct + * + * This handler brings down the global pstate closer to the local pstate + * according quadratic equation. Queues a new timer if it is still not equal + * to local pstate + */ +static void gpstate_timer_handler(struct timer_list *t) +{ + struct global_pstate_info *gpstates = timer_container_of(gpstates, t, + timer); + struct cpufreq_policy *policy = gpstates->policy; + int gpstate_idx, lpstate_idx; + unsigned long val; + unsigned int time_diff = jiffies_to_msecs(jiffies) + - gpstates->last_sampled_time; + struct powernv_smp_call_data freq_data; + + if (!spin_trylock(&gpstates->gpstate_lock)) + return; + /* + * If the timer has migrated to the different cpu then bring + * it back to one of the policy->cpus + */ + if (!cpumask_test_cpu(raw_smp_processor_id(), policy->cpus)) { + gpstates->timer.expires = jiffies + msecs_to_jiffies(1); + add_timer_on(&gpstates->timer, cpumask_first(policy->cpus)); + spin_unlock(&gpstates->gpstate_lock); + return; + } + + /* + * If PMCR was last updated was using fast_switch then + * We may have wrong in gpstate->last_lpstate_idx + * value. Hence, read from PMCR to get correct data. + */ + val = get_pmspr(SPRN_PMCR); + freq_data.gpstate_id = extract_global_pstate(val); + freq_data.pstate_id = extract_local_pstate(val); + if (freq_data.gpstate_id == freq_data.pstate_id) { + reset_gpstates(policy); + spin_unlock(&gpstates->gpstate_lock); + return; + } + + gpstates->last_sampled_time += time_diff; + gpstates->elapsed_time += time_diff; + + if (gpstates->elapsed_time > MAX_RAMP_DOWN_TIME) { + gpstate_idx = pstate_to_idx(freq_data.pstate_id); + lpstate_idx = gpstate_idx; + reset_gpstates(policy); + gpstates->highest_lpstate_idx = gpstate_idx; + } else { + lpstate_idx = pstate_to_idx(freq_data.pstate_id); + gpstate_idx = calc_global_pstate(gpstates->elapsed_time, + gpstates->highest_lpstate_idx, + lpstate_idx); + } + freq_data.gpstate_id = idx_to_pstate(gpstate_idx); + gpstates->last_gpstate_idx = gpstate_idx; + gpstates->last_lpstate_idx = lpstate_idx; + /* + * If local pstate is equal to global pstate, rampdown is over + * So timer is not required to be queued. + */ + if (gpstate_idx != gpstates->last_lpstate_idx) + queue_gpstate_timer(gpstates); + + set_pstate(&freq_data); + spin_unlock(&gpstates->gpstate_lock); +} + +/* + * powernv_cpufreq_target_index: Sets the frequency corresponding to + * the cpufreq table entry indexed by new_index on the cpus in the + * mask policy->cpus + */ +static int powernv_cpufreq_target_index(struct cpufreq_policy *policy, + unsigned int new_index) +{ + struct powernv_smp_call_data freq_data; + unsigned int cur_msec, gpstate_idx; + struct global_pstate_info *gpstates = policy->driver_data; + + if (unlikely(rebooting) && new_index != get_nominal_index()) + return 0; + + if (!throttled) { + /* we don't want to be preempted while + * checking if the CPU frequency has been throttled + */ + preempt_disable(); + powernv_cpufreq_throttle_check(NULL); + preempt_enable(); + } + + cur_msec = jiffies_to_msecs(get_jiffies_64()); + + freq_data.pstate_id = idx_to_pstate(new_index); + if (!gpstates) { + freq_data.gpstate_id = freq_data.pstate_id; + goto no_gpstate; + } + + spin_lock(&gpstates->gpstate_lock); + + if (!gpstates->last_sampled_time) { + gpstate_idx = new_index; + gpstates->highest_lpstate_idx = new_index; + goto gpstates_done; + } + + if (gpstates->last_gpstate_idx < new_index) { + gpstates->elapsed_time += cur_msec - + gpstates->last_sampled_time; + + /* + * If its has been ramping down for more than MAX_RAMP_DOWN_TIME + * we should be resetting all global pstate related data. Set it + * equal to local pstate to start fresh. + */ + if (gpstates->elapsed_time > MAX_RAMP_DOWN_TIME) { + reset_gpstates(policy); + gpstates->highest_lpstate_idx = new_index; + gpstate_idx = new_index; + } else { + /* Elaspsed_time is less than 5 seconds, continue to rampdown */ + gpstate_idx = calc_global_pstate(gpstates->elapsed_time, + gpstates->highest_lpstate_idx, + new_index); + } + } else { + reset_gpstates(policy); + gpstates->highest_lpstate_idx = new_index; + gpstate_idx = new_index; + } + + /* + * If local pstate is equal to global pstate, rampdown is over + * So timer is not required to be queued. + */ + if (gpstate_idx != new_index) + queue_gpstate_timer(gpstates); + else + timer_delete_sync(&gpstates->timer); + +gpstates_done: + freq_data.gpstate_id = idx_to_pstate(gpstate_idx); + gpstates->last_sampled_time = cur_msec; + gpstates->last_gpstate_idx = gpstate_idx; + gpstates->last_lpstate_idx = new_index; + + spin_unlock(&gpstates->gpstate_lock); + +no_gpstate: + /* + * Use smp_call_function to send IPI and execute the + * mtspr on target CPU. We could do that without IPI + * if current CPU is within policy->cpus (core) + */ + smp_call_function_any(policy->cpus, set_pstate, &freq_data, 1); + return 0; +} + +static int powernv_cpufreq_cpu_init(struct cpufreq_policy *policy) +{ + int base, i; + struct kernfs_node *kn; + struct global_pstate_info *gpstates; + + base = cpu_first_thread_sibling(policy->cpu); + + for (i = 0; i < threads_per_core; i++) + cpumask_set_cpu(base + i, policy->cpus); + + kn = kernfs_find_and_get(policy->kobj.sd, throttle_attr_grp.name); + if (!kn) { + int ret; + + ret = sysfs_create_group(&policy->kobj, &throttle_attr_grp); + if (ret) { + pr_info("Failed to create throttle stats directory for cpu %d\n", + policy->cpu); + return ret; + } + } else { + kernfs_put(kn); + } + + policy->freq_table = powernv_freqs; + policy->fast_switch_possible = true; + + if (pvr_version_is(PVR_POWER9)) + return 0; + + /* Initialise Gpstate ramp-down timer only on POWER8 */ + gpstates = kzalloc(sizeof(*gpstates), GFP_KERNEL); + if (!gpstates) + return -ENOMEM; + + policy->driver_data = gpstates; + + /* initialize timer */ + gpstates->policy = policy; + timer_setup(&gpstates->timer, gpstate_timer_handler, + TIMER_PINNED | TIMER_DEFERRABLE); + gpstates->timer.expires = jiffies + + msecs_to_jiffies(GPSTATE_TIMER_INTERVAL); + spin_lock_init(&gpstates->gpstate_lock); + + return 0; +} + +static void powernv_cpufreq_cpu_exit(struct cpufreq_policy *policy) +{ + struct powernv_smp_call_data freq_data; + struct global_pstate_info *gpstates = policy->driver_data; + + freq_data.pstate_id = idx_to_pstate(powernv_pstate_info.min); + freq_data.gpstate_id = idx_to_pstate(powernv_pstate_info.min); + smp_call_function_single(policy->cpu, set_pstate, &freq_data, 1); + if (gpstates) + timer_delete_sync(&gpstates->timer); + + kfree(policy->driver_data); +} + +static int powernv_cpufreq_reboot_notifier(struct notifier_block *nb, + unsigned long action, void *unused) +{ + int cpu; + struct cpufreq_policy *cpu_policy; + + rebooting = true; + for_each_online_cpu(cpu) { + cpu_policy = cpufreq_cpu_get(cpu); + if (!cpu_policy) + continue; + powernv_cpufreq_target_index(cpu_policy, get_nominal_index()); + cpufreq_cpu_put(cpu_policy); + } + + return NOTIFY_DONE; +} + +static struct notifier_block powernv_cpufreq_reboot_nb = { + .notifier_call = powernv_cpufreq_reboot_notifier, +}; + +static void powernv_cpufreq_work_fn(struct work_struct *work) +{ + struct chip *chip = container_of(work, struct chip, throttle); + struct cpufreq_policy *policy; + unsigned int cpu; + cpumask_t mask; + + cpus_read_lock(); + cpumask_and(&mask, &chip->mask, cpu_online_mask); + smp_call_function_any(&mask, + powernv_cpufreq_throttle_check, NULL, 0); + + if (!chip->restore) + goto out; + + chip->restore = false; + for_each_cpu(cpu, &mask) { + int index; + + policy = cpufreq_cpu_get(cpu); + if (!policy) + continue; + index = cpufreq_table_find_index_c(policy, policy->cur, false); + powernv_cpufreq_target_index(policy, index); + cpumask_andnot(&mask, &mask, policy->cpus); + cpufreq_cpu_put(policy); + } +out: + cpus_read_unlock(); +} + +static int powernv_cpufreq_occ_msg(struct notifier_block *nb, + unsigned long msg_type, void *_msg) +{ + struct opal_msg *msg = _msg; + struct opal_occ_msg omsg; + int i; + + if (msg_type != OPAL_MSG_OCC) + return 0; + + omsg.type = be64_to_cpu(msg->params[0]); + + switch (omsg.type) { + case OCC_RESET: + occ_reset = true; + pr_info("OCC (On Chip Controller - enforces hard thermal/power limits) Resetting\n"); + /* + * powernv_cpufreq_throttle_check() is called in + * target() callback which can detect the throttle state + * for governors like ondemand. + * But static governors will not call target() often thus + * report throttling here. + */ + if (!throttled) { + throttled = true; + pr_warn("CPU frequency is throttled for duration\n"); + } + + break; + case OCC_LOAD: + pr_info("OCC Loading, CPU frequency is throttled until OCC is started\n"); + break; + case OCC_THROTTLE: + omsg.chip = be64_to_cpu(msg->params[1]); + omsg.throttle_status = be64_to_cpu(msg->params[2]); + + if (occ_reset) { + occ_reset = false; + throttled = false; + pr_info("OCC Active, CPU frequency is no longer throttled\n"); + + for (i = 0; i < nr_chips; i++) { + chips[i].restore = true; + schedule_work(&chips[i].throttle); + } + + return 0; + } + + for (i = 0; i < nr_chips; i++) + if (chips[i].id == omsg.chip) + break; + + if (omsg.throttle_status >= 0 && + omsg.throttle_status <= OCC_MAX_THROTTLE_STATUS) { + chips[i].throttle_reason = omsg.throttle_status; + chips[i].reason[omsg.throttle_status]++; + } + + if (!omsg.throttle_status) + chips[i].restore = true; + + schedule_work(&chips[i].throttle); + } + return 0; +} + +static struct notifier_block powernv_cpufreq_opal_nb = { + .notifier_call = powernv_cpufreq_occ_msg, + .next = NULL, + .priority = 0, +}; + +static unsigned int powernv_fast_switch(struct cpufreq_policy *policy, + unsigned int target_freq) +{ + int index; + struct powernv_smp_call_data freq_data; + + index = cpufreq_table_find_index_dl(policy, target_freq, false); + freq_data.pstate_id = powernv_freqs[index].driver_data; + freq_data.gpstate_id = powernv_freqs[index].driver_data; + set_pstate(&freq_data); + + return powernv_freqs[index].frequency; +} + +static struct cpufreq_driver powernv_cpufreq_driver = { + .name = "powernv-cpufreq", + .flags = CPUFREQ_CONST_LOOPS, + .init = powernv_cpufreq_cpu_init, + .exit = powernv_cpufreq_cpu_exit, + .verify = cpufreq_generic_frequency_table_verify, + .target_index = powernv_cpufreq_target_index, + .fast_switch = powernv_fast_switch, + .get = powernv_cpufreq_get, + .attr = powernv_cpu_freq_attr, +}; + +static int init_chip_info(void) +{ + unsigned int *chip; + unsigned int cpu, i; + unsigned int prev_chip_id = UINT_MAX; + cpumask_t *chip_cpu_mask; + int ret = 0; + + chip = kcalloc(num_possible_cpus(), sizeof(*chip), GFP_KERNEL); + if (!chip) + return -ENOMEM; + + /* Allocate a chip cpu mask large enough to fit mask for all chips */ + chip_cpu_mask = kcalloc(MAX_NR_CHIPS, sizeof(cpumask_t), GFP_KERNEL); + if (!chip_cpu_mask) { + ret = -ENOMEM; + goto free_and_return; + } + + for_each_possible_cpu(cpu) { + unsigned int id = cpu_to_chip_id(cpu); + + if (prev_chip_id != id) { + prev_chip_id = id; + chip[nr_chips++] = id; + } + cpumask_set_cpu(cpu, &chip_cpu_mask[nr_chips-1]); + } + + chips = kcalloc(nr_chips, sizeof(struct chip), GFP_KERNEL); + if (!chips) { + ret = -ENOMEM; + goto out_free_chip_cpu_mask; + } + + for (i = 0; i < nr_chips; i++) { + chips[i].id = chip[i]; + cpumask_copy(&chips[i].mask, &chip_cpu_mask[i]); + INIT_WORK(&chips[i].throttle, powernv_cpufreq_work_fn); + for_each_cpu(cpu, &chips[i].mask) + per_cpu(chip_info, cpu) = &chips[i]; + } + +out_free_chip_cpu_mask: + kfree(chip_cpu_mask); +free_and_return: + kfree(chip); + return ret; +} + +static inline void clean_chip_info(void) +{ + int i; + + /* flush any pending work items */ + if (chips) + for (i = 0; i < nr_chips; i++) + cancel_work_sync(&chips[i].throttle); + kfree(chips); +} + +static inline void unregister_all_notifiers(void) +{ + opal_message_notifier_unregister(OPAL_MSG_OCC, + &powernv_cpufreq_opal_nb); + unregister_reboot_notifier(&powernv_cpufreq_reboot_nb); +} + +static int __init powernv_cpufreq_init(void) +{ + int rc = 0; + + /* Don't probe on pseries (guest) platforms */ + if (!firmware_has_feature(FW_FEATURE_OPAL)) + return -ENODEV; + + /* Discover pstates from device tree and init */ + rc = init_powernv_pstates(); + if (rc) + goto out; + + /* Populate chip info */ + rc = init_chip_info(); + if (rc) + goto out; + + if (powernv_pstate_info.wof_enabled) + powernv_cpufreq_driver.set_boost = cpufreq_boost_set_sw; + + rc = cpufreq_register_driver(&powernv_cpufreq_driver); + if (rc) { + pr_info("Failed to register the cpufreq driver (%d)\n", rc); + goto cleanup; + } + + register_reboot_notifier(&powernv_cpufreq_reboot_nb); + opal_message_notifier_register(OPAL_MSG_OCC, &powernv_cpufreq_opal_nb); + + return 0; +cleanup: + clean_chip_info(); +out: + pr_info("Platform driver disabled. System does not support PState control\n"); + return rc; +} +module_init(powernv_cpufreq_init); + +static void __exit powernv_cpufreq_exit(void) +{ + cpufreq_unregister_driver(&powernv_cpufreq_driver); + unregister_all_notifiers(); + clean_chip_info(); +} +module_exit(powernv_cpufreq_exit); + +MODULE_DESCRIPTION("cpufreq driver for IBM/OpenPOWER powernv systems"); +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Vaidyanathan Srinivasan <svaidy at linux.vnet.ibm.com>"); diff --git a/drivers/cpufreq/powernv-trace.h b/drivers/cpufreq/powernv-trace.h new file mode 100644 index 000000000000..8cadb7c9427b --- /dev/null +++ b/drivers/cpufreq/powernv-trace.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#if !defined(_POWERNV_TRACE_H) || defined(TRACE_HEADER_MULTI_READ) +#define _POWERNV_TRACE_H + +#include <linux/cpufreq.h> +#include <linux/tracepoint.h> +#include <linux/trace_events.h> + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM power + +TRACE_EVENT(powernv_throttle, + + TP_PROTO(int chip_id, const char *reason, int pmax), + + TP_ARGS(chip_id, reason, pmax), + + TP_STRUCT__entry( + __field(int, chip_id) + __string(reason, reason) + __field(int, pmax) + ), + + TP_fast_assign( + __entry->chip_id = chip_id; + __assign_str(reason); + __entry->pmax = pmax; + ), + + TP_printk("Chip %d Pmax %d %s", __entry->chip_id, + __entry->pmax, __get_str(reason)) +); + +#endif /* _POWERNV_TRACE_H */ + +/* This part must be outside protection */ +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . + +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE powernv-trace + +#include <trace/define_trace.h> diff --git a/drivers/cpufreq/ppc-corenet-cpufreq.c b/drivers/cpufreq/ppc-corenet-cpufreq.c deleted file mode 100644 index 3cae4529f959..000000000000 --- a/drivers/cpufreq/ppc-corenet-cpufreq.c +++ /dev/null @@ -1,380 +0,0 @@ -/* - * Copyright 2013 Freescale Semiconductor, Inc. - * - * CPU Frequency Scaling driver for Freescale PowerPC corenet SoCs. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - */ - -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - -#include <linux/clk.h> -#include <linux/cpufreq.h> -#include <linux/errno.h> -#include <sysdev/fsl_soc.h> -#include <linux/init.h> -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/mutex.h> -#include <linux/of.h> -#include <linux/slab.h> -#include <linux/smp.h> - -/** - * struct cpu_data - per CPU data struct - * @clk: the clk of CPU - * @parent: the parent node of cpu clock - * @table: frequency table - */ -struct cpu_data { - struct clk *clk; - struct device_node *parent; - struct cpufreq_frequency_table *table; -}; - -/** - * struct soc_data - SoC specific data - * @freq_mask: mask the disallowed frequencies - * @flag: unique flags - */ -struct soc_data { - u32 freq_mask[4]; - u32 flag; -}; - -#define FREQ_MASK 1 -/* see hardware specification for the allowed frqeuencies */ -static const struct soc_data sdata[] = { - { /* used by p2041 and p3041 */ - .freq_mask = {0x8, 0x8, 0x2, 0x2}, - .flag = FREQ_MASK, - }, - { /* used by p5020 */ - .freq_mask = {0x8, 0x2}, - .flag = FREQ_MASK, - }, - { /* used by p4080, p5040 */ - .freq_mask = {0}, - .flag = 0, - }, -}; - -/* - * the minimum allowed core frequency, in Hz - * for chassis v1.0, >= platform frequency - * for chassis v2.0, >= platform frequency / 2 - */ -static u32 min_cpufreq; -static const u32 *fmask; - -/* serialize frequency changes */ -static DEFINE_MUTEX(cpufreq_lock); -static DEFINE_PER_CPU(struct cpu_data *, cpu_data); - -/* cpumask in a cluster */ -static DEFINE_PER_CPU(cpumask_var_t, cpu_mask); - -#ifndef CONFIG_SMP -static inline const struct cpumask *cpu_core_mask(int cpu) -{ - return cpumask_of(0); -} -#endif - -static unsigned int corenet_cpufreq_get_speed(unsigned int cpu) -{ - struct cpu_data *data = per_cpu(cpu_data, cpu); - - return clk_get_rate(data->clk) / 1000; -} - -/* reduce the duplicated frequencies in frequency table */ -static void freq_table_redup(struct cpufreq_frequency_table *freq_table, - int count) -{ - int i, j; - - for (i = 1; i < count; i++) { - for (j = 0; j < i; j++) { - if (freq_table[j].frequency == CPUFREQ_ENTRY_INVALID || - freq_table[j].frequency != - freq_table[i].frequency) - continue; - - freq_table[i].frequency = CPUFREQ_ENTRY_INVALID; - break; - } - } -} - -/* sort the frequencies in frequency table in descenting order */ -static void freq_table_sort(struct cpufreq_frequency_table *freq_table, - int count) -{ - int i, j, ind; - unsigned int freq, max_freq; - struct cpufreq_frequency_table table; - for (i = 0; i < count - 1; i++) { - max_freq = freq_table[i].frequency; - ind = i; - for (j = i + 1; j < count; j++) { - freq = freq_table[j].frequency; - if (freq == CPUFREQ_ENTRY_INVALID || - freq <= max_freq) - continue; - ind = j; - max_freq = freq; - } - - if (ind != i) { - /* exchange the frequencies */ - table.driver_data = freq_table[i].driver_data; - table.frequency = freq_table[i].frequency; - freq_table[i].driver_data = freq_table[ind].driver_data; - freq_table[i].frequency = freq_table[ind].frequency; - freq_table[ind].driver_data = table.driver_data; - freq_table[ind].frequency = table.frequency; - } - } -} - -static int corenet_cpufreq_cpu_init(struct cpufreq_policy *policy) -{ - struct device_node *np; - int i, count, ret; - u32 freq, mask; - struct clk *clk; - struct cpufreq_frequency_table *table; - struct cpu_data *data; - unsigned int cpu = policy->cpu; - - np = of_get_cpu_node(cpu, NULL); - if (!np) - return -ENODEV; - - data = kzalloc(sizeof(*data), GFP_KERNEL); - if (!data) { - pr_err("%s: no memory\n", __func__); - goto err_np; - } - - data->clk = of_clk_get(np, 0); - if (IS_ERR(data->clk)) { - pr_err("%s: no clock information\n", __func__); - goto err_nomem2; - } - - data->parent = of_parse_phandle(np, "clocks", 0); - if (!data->parent) { - pr_err("%s: could not get clock information\n", __func__); - goto err_nomem2; - } - - count = of_property_count_strings(data->parent, "clock-names"); - table = kcalloc(count + 1, sizeof(*table), GFP_KERNEL); - if (!table) { - pr_err("%s: no memory\n", __func__); - goto err_node; - } - - if (fmask) - mask = fmask[get_hard_smp_processor_id(cpu)]; - else - mask = 0x0; - - for (i = 0; i < count; i++) { - clk = of_clk_get(data->parent, i); - freq = clk_get_rate(clk); - /* - * the clock is valid if its frequency is not masked - * and large than minimum allowed frequency. - */ - if (freq < min_cpufreq || (mask & (1 << i))) - table[i].frequency = CPUFREQ_ENTRY_INVALID; - else - table[i].frequency = freq / 1000; - table[i].driver_data = i; - } - freq_table_redup(table, count); - freq_table_sort(table, count); - table[i].frequency = CPUFREQ_TABLE_END; - - /* set the min and max frequency properly */ - ret = cpufreq_frequency_table_cpuinfo(policy, table); - if (ret) { - pr_err("invalid frequency table: %d\n", ret); - goto err_nomem1; - } - - data->table = table; - per_cpu(cpu_data, cpu) = data; - - /* update ->cpus if we have cluster, no harm if not */ - cpumask_copy(policy->cpus, per_cpu(cpu_mask, cpu)); - for_each_cpu(i, per_cpu(cpu_mask, cpu)) - per_cpu(cpu_data, i) = data; - - policy->cpuinfo.transition_latency = CPUFREQ_ETERNAL; - policy->cur = corenet_cpufreq_get_speed(policy->cpu); - - cpufreq_frequency_table_get_attr(table, cpu); - of_node_put(np); - - return 0; - -err_nomem1: - kfree(table); -err_node: - of_node_put(data->parent); -err_nomem2: - per_cpu(cpu_data, cpu) = NULL; - kfree(data); -err_np: - of_node_put(np); - - return -ENODEV; -} - -static int __exit corenet_cpufreq_cpu_exit(struct cpufreq_policy *policy) -{ - struct cpu_data *data = per_cpu(cpu_data, policy->cpu); - unsigned int cpu; - - cpufreq_frequency_table_put_attr(policy->cpu); - of_node_put(data->parent); - kfree(data->table); - kfree(data); - - for_each_cpu(cpu, per_cpu(cpu_mask, policy->cpu)) - per_cpu(cpu_data, cpu) = NULL; - - return 0; -} - -static int corenet_cpufreq_verify(struct cpufreq_policy *policy) -{ - struct cpufreq_frequency_table *table = - per_cpu(cpu_data, policy->cpu)->table; - - return cpufreq_frequency_table_verify(policy, table); -} - -static int corenet_cpufreq_target(struct cpufreq_policy *policy, - unsigned int target_freq, unsigned int relation) -{ - struct cpufreq_freqs freqs; - unsigned int new; - struct clk *parent; - int ret; - struct cpu_data *data = per_cpu(cpu_data, policy->cpu); - - cpufreq_frequency_table_target(policy, data->table, - target_freq, relation, &new); - - if (policy->cur == data->table[new].frequency) - return 0; - - freqs.old = policy->cur; - freqs.new = data->table[new].frequency; - - mutex_lock(&cpufreq_lock); - cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE); - - parent = of_clk_get(data->parent, data->table[new].driver_data); - ret = clk_set_parent(data->clk, parent); - if (ret) - freqs.new = freqs.old; - - cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); - mutex_unlock(&cpufreq_lock); - - return ret; -} - -static struct freq_attr *corenet_cpufreq_attr[] = { - &cpufreq_freq_attr_scaling_available_freqs, - NULL, -}; - -static struct cpufreq_driver ppc_corenet_cpufreq_driver = { - .name = "ppc_cpufreq", - .owner = THIS_MODULE, - .flags = CPUFREQ_CONST_LOOPS, - .init = corenet_cpufreq_cpu_init, - .exit = __exit_p(corenet_cpufreq_cpu_exit), - .verify = corenet_cpufreq_verify, - .target = corenet_cpufreq_target, - .get = corenet_cpufreq_get_speed, - .attr = corenet_cpufreq_attr, -}; - -static const struct of_device_id node_matches[] __initdata = { - { .compatible = "fsl,p2041-clockgen", .data = &sdata[0], }, - { .compatible = "fsl,p3041-clockgen", .data = &sdata[0], }, - { .compatible = "fsl,p5020-clockgen", .data = &sdata[1], }, - { .compatible = "fsl,p4080-clockgen", .data = &sdata[2], }, - { .compatible = "fsl,p5040-clockgen", .data = &sdata[2], }, - { .compatible = "fsl,qoriq-clockgen-2.0", }, - {} -}; - -static int __init ppc_corenet_cpufreq_init(void) -{ - int ret; - struct device_node *np; - const struct of_device_id *match; - const struct soc_data *data; - unsigned int cpu; - - np = of_find_matching_node(NULL, node_matches); - if (!np) - return -ENODEV; - - for_each_possible_cpu(cpu) { - if (!alloc_cpumask_var(&per_cpu(cpu_mask, cpu), GFP_KERNEL)) - goto err_mask; - cpumask_copy(per_cpu(cpu_mask, cpu), cpu_core_mask(cpu)); - } - - match = of_match_node(node_matches, np); - data = match->data; - if (data) { - if (data->flag) - fmask = data->freq_mask; - min_cpufreq = fsl_get_sys_freq(); - } else { - min_cpufreq = fsl_get_sys_freq() / 2; - } - - of_node_put(np); - - ret = cpufreq_register_driver(&ppc_corenet_cpufreq_driver); - if (!ret) - pr_info("Freescale PowerPC corenet CPU frequency scaling driver\n"); - - return ret; - -err_mask: - for_each_possible_cpu(cpu) - free_cpumask_var(per_cpu(cpu_mask, cpu)); - - return -ENOMEM; -} -module_init(ppc_corenet_cpufreq_init); - -static void __exit ppc_corenet_cpufreq_exit(void) -{ - unsigned int cpu; - - for_each_possible_cpu(cpu) - free_cpumask_var(per_cpu(cpu_mask, cpu)); - - cpufreq_unregister_driver(&ppc_corenet_cpufreq_driver); -} -module_exit(ppc_corenet_cpufreq_exit); - -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Tang Yuantian <Yuantian.Tang@freescale.com>"); -MODULE_DESCRIPTION("cpufreq driver for Freescale e500mc series SoCs"); diff --git a/drivers/cpufreq/ppc_cbe_cpufreq.c b/drivers/cpufreq/ppc_cbe_cpufreq.c deleted file mode 100644 index 5936f8d6f2cc..000000000000 --- a/drivers/cpufreq/ppc_cbe_cpufreq.c +++ /dev/null @@ -1,209 +0,0 @@ -/* - * cpufreq driver for the cell processor - * - * (C) Copyright IBM Deutschland Entwicklung GmbH 2005-2007 - * - * Author: Christian Krafft <krafft@de.ibm.com> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2, or (at your option) - * any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - */ - -#include <linux/cpufreq.h> -#include <linux/module.h> -#include <linux/of_platform.h> - -#include <asm/machdep.h> -#include <asm/prom.h> -#include <asm/cell-regs.h> - -#include "ppc_cbe_cpufreq.h" - -static DEFINE_MUTEX(cbe_switch_mutex); - - -/* the CBE supports an 8 step frequency scaling */ -static struct cpufreq_frequency_table cbe_freqs[] = { - {1, 0}, - {2, 0}, - {3, 0}, - {4, 0}, - {5, 0}, - {6, 0}, - {8, 0}, - {10, 0}, - {0, CPUFREQ_TABLE_END}, -}; - -/* - * hardware specific functions - */ - -static int set_pmode(unsigned int cpu, unsigned int slow_mode) -{ - int rc; - - if (cbe_cpufreq_has_pmi) - rc = cbe_cpufreq_set_pmode_pmi(cpu, slow_mode); - else - rc = cbe_cpufreq_set_pmode(cpu, slow_mode); - - pr_debug("register contains slow mode %d\n", cbe_cpufreq_get_pmode(cpu)); - - return rc; -} - -/* - * cpufreq functions - */ - -static int cbe_cpufreq_cpu_init(struct cpufreq_policy *policy) -{ - const u32 *max_freqp; - u32 max_freq; - int i, cur_pmode; - struct device_node *cpu; - - cpu = of_get_cpu_node(policy->cpu, NULL); - - if (!cpu) - return -ENODEV; - - pr_debug("init cpufreq on CPU %d\n", policy->cpu); - - /* - * Let's check we can actually get to the CELL regs - */ - if (!cbe_get_cpu_pmd_regs(policy->cpu) || - !cbe_get_cpu_mic_tm_regs(policy->cpu)) { - pr_info("invalid CBE regs pointers for cpufreq\n"); - return -EINVAL; - } - - max_freqp = of_get_property(cpu, "clock-frequency", NULL); - - of_node_put(cpu); - - if (!max_freqp) - return -EINVAL; - - /* we need the freq in kHz */ - max_freq = *max_freqp / 1000; - - pr_debug("max clock-frequency is at %u kHz\n", max_freq); - pr_debug("initializing frequency table\n"); - - /* initialize frequency table */ - for (i=0; cbe_freqs[i].frequency!=CPUFREQ_TABLE_END; i++) { - cbe_freqs[i].frequency = max_freq / cbe_freqs[i].driver_data; - pr_debug("%d: %d\n", i, cbe_freqs[i].frequency); - } - - /* if DEBUG is enabled set_pmode() measures the latency - * of a transition */ - policy->cpuinfo.transition_latency = 25000; - - cur_pmode = cbe_cpufreq_get_pmode(policy->cpu); - pr_debug("current pmode is at %d\n",cur_pmode); - - policy->cur = cbe_freqs[cur_pmode].frequency; - -#ifdef CONFIG_SMP - cpumask_copy(policy->cpus, cpu_sibling_mask(policy->cpu)); -#endif - - cpufreq_frequency_table_get_attr(cbe_freqs, policy->cpu); - - /* this ensures that policy->cpuinfo_min - * and policy->cpuinfo_max are set correctly */ - return cpufreq_frequency_table_cpuinfo(policy, cbe_freqs); -} - -static int cbe_cpufreq_cpu_exit(struct cpufreq_policy *policy) -{ - cpufreq_frequency_table_put_attr(policy->cpu); - return 0; -} - -static int cbe_cpufreq_verify(struct cpufreq_policy *policy) -{ - return cpufreq_frequency_table_verify(policy, cbe_freqs); -} - -static int cbe_cpufreq_target(struct cpufreq_policy *policy, - unsigned int target_freq, - unsigned int relation) -{ - int rc; - struct cpufreq_freqs freqs; - unsigned int cbe_pmode_new; - - cpufreq_frequency_table_target(policy, - cbe_freqs, - target_freq, - relation, - &cbe_pmode_new); - - freqs.old = policy->cur; - freqs.new = cbe_freqs[cbe_pmode_new].frequency; - - mutex_lock(&cbe_switch_mutex); - cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE); - - pr_debug("setting frequency for cpu %d to %d kHz, " \ - "1/%d of max frequency\n", - policy->cpu, - cbe_freqs[cbe_pmode_new].frequency, - cbe_freqs[cbe_pmode_new].driver_data); - - rc = set_pmode(policy->cpu, cbe_pmode_new); - - cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); - mutex_unlock(&cbe_switch_mutex); - - return rc; -} - -static struct cpufreq_driver cbe_cpufreq_driver = { - .verify = cbe_cpufreq_verify, - .target = cbe_cpufreq_target, - .init = cbe_cpufreq_cpu_init, - .exit = cbe_cpufreq_cpu_exit, - .name = "cbe-cpufreq", - .owner = THIS_MODULE, - .flags = CPUFREQ_CONST_LOOPS, -}; - -/* - * module init and destoy - */ - -static int __init cbe_cpufreq_init(void) -{ - if (!machine_is(cell)) - return -ENODEV; - - return cpufreq_register_driver(&cbe_cpufreq_driver); -} - -static void __exit cbe_cpufreq_exit(void) -{ - cpufreq_unregister_driver(&cbe_cpufreq_driver); -} - -module_init(cbe_cpufreq_init); -module_exit(cbe_cpufreq_exit); - -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Christian Krafft <krafft@de.ibm.com>"); diff --git a/drivers/cpufreq/ppc_cbe_cpufreq.h b/drivers/cpufreq/ppc_cbe_cpufreq.h deleted file mode 100644 index b4c00a5a6a59..000000000000 --- a/drivers/cpufreq/ppc_cbe_cpufreq.h +++ /dev/null @@ -1,24 +0,0 @@ -/* - * ppc_cbe_cpufreq.h - * - * This file contains the definitions used by the cbe_cpufreq driver. - * - * (C) Copyright IBM Deutschland Entwicklung GmbH 2005-2007 - * - * Author: Christian Krafft <krafft@de.ibm.com> - * - */ - -#include <linux/cpufreq.h> -#include <linux/types.h> - -int cbe_cpufreq_set_pmode(int cpu, unsigned int pmode); -int cbe_cpufreq_get_pmode(int cpu); - -int cbe_cpufreq_set_pmode_pmi(int cpu, unsigned int pmode); - -#if defined(CONFIG_CPU_FREQ_CBE_PMI) || defined(CONFIG_CPU_FREQ_CBE_PMI_MODULE) -extern bool cbe_cpufreq_has_pmi; -#else -#define cbe_cpufreq_has_pmi (0) -#endif diff --git a/drivers/cpufreq/ppc_cbe_cpufreq_pervasive.c b/drivers/cpufreq/ppc_cbe_cpufreq_pervasive.c deleted file mode 100644 index 84d2f2cf5ba7..000000000000 --- a/drivers/cpufreq/ppc_cbe_cpufreq_pervasive.c +++ /dev/null @@ -1,115 +0,0 @@ -/* - * pervasive backend for the cbe_cpufreq driver - * - * This driver makes use of the pervasive unit to - * engage the desired frequency. - * - * (C) Copyright IBM Deutschland Entwicklung GmbH 2005-2007 - * - * Author: Christian Krafft <krafft@de.ibm.com> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2, or (at your option) - * any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - */ - -#include <linux/io.h> -#include <linux/kernel.h> -#include <linux/time.h> -#include <asm/machdep.h> -#include <asm/hw_irq.h> -#include <asm/cell-regs.h> - -#include "ppc_cbe_cpufreq.h" - -/* to write to MIC register */ -static u64 MIC_Slow_Fast_Timer_table[] = { - [0 ... 7] = 0x007fc00000000000ull, -}; - -/* more values for the MIC */ -static u64 MIC_Slow_Next_Timer_table[] = { - 0x0000240000000000ull, - 0x0000268000000000ull, - 0x000029C000000000ull, - 0x00002D0000000000ull, - 0x0000300000000000ull, - 0x0000334000000000ull, - 0x000039C000000000ull, - 0x00003FC000000000ull, -}; - - -int cbe_cpufreq_set_pmode(int cpu, unsigned int pmode) -{ - struct cbe_pmd_regs __iomem *pmd_regs; - struct cbe_mic_tm_regs __iomem *mic_tm_regs; - unsigned long flags; - u64 value; -#ifdef DEBUG - long time; -#endif - - local_irq_save(flags); - - mic_tm_regs = cbe_get_cpu_mic_tm_regs(cpu); - pmd_regs = cbe_get_cpu_pmd_regs(cpu); - -#ifdef DEBUG - time = jiffies; -#endif - - out_be64(&mic_tm_regs->slow_fast_timer_0, MIC_Slow_Fast_Timer_table[pmode]); - out_be64(&mic_tm_regs->slow_fast_timer_1, MIC_Slow_Fast_Timer_table[pmode]); - - out_be64(&mic_tm_regs->slow_next_timer_0, MIC_Slow_Next_Timer_table[pmode]); - out_be64(&mic_tm_regs->slow_next_timer_1, MIC_Slow_Next_Timer_table[pmode]); - - value = in_be64(&pmd_regs->pmcr); - /* set bits to zero */ - value &= 0xFFFFFFFFFFFFFFF8ull; - /* set bits to next pmode */ - value |= pmode; - - out_be64(&pmd_regs->pmcr, value); - -#ifdef DEBUG - /* wait until new pmode appears in status register */ - value = in_be64(&pmd_regs->pmsr) & 0x07; - while (value != pmode) { - cpu_relax(); - value = in_be64(&pmd_regs->pmsr) & 0x07; - } - - time = jiffies - time; - time = jiffies_to_msecs(time); - pr_debug("had to wait %lu ms for a transition using " \ - "pervasive unit\n", time); -#endif - local_irq_restore(flags); - - return 0; -} - - -int cbe_cpufreq_get_pmode(int cpu) -{ - int ret; - struct cbe_pmd_regs __iomem *pmd_regs; - - pmd_regs = cbe_get_cpu_pmd_regs(cpu); - ret = in_be64(&pmd_regs->pmsr) & 0x07; - - return ret; -} - diff --git a/drivers/cpufreq/ppc_cbe_cpufreq_pmi.c b/drivers/cpufreq/ppc_cbe_cpufreq_pmi.c deleted file mode 100644 index d29e8da396a0..000000000000 --- a/drivers/cpufreq/ppc_cbe_cpufreq_pmi.c +++ /dev/null @@ -1,156 +0,0 @@ -/* - * pmi backend for the cbe_cpufreq driver - * - * (C) Copyright IBM Deutschland Entwicklung GmbH 2005-2007 - * - * Author: Christian Krafft <krafft@de.ibm.com> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2, or (at your option) - * any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - */ - -#include <linux/kernel.h> -#include <linux/types.h> -#include <linux/timer.h> -#include <linux/module.h> -#include <linux/of_platform.h> - -#include <asm/processor.h> -#include <asm/prom.h> -#include <asm/pmi.h> -#include <asm/cell-regs.h> - -#ifdef DEBUG -#include <asm/time.h> -#endif - -#include "ppc_cbe_cpufreq.h" - -static u8 pmi_slow_mode_limit[MAX_CBE]; - -bool cbe_cpufreq_has_pmi = false; -EXPORT_SYMBOL_GPL(cbe_cpufreq_has_pmi); - -/* - * hardware specific functions - */ - -int cbe_cpufreq_set_pmode_pmi(int cpu, unsigned int pmode) -{ - int ret; - pmi_message_t pmi_msg; -#ifdef DEBUG - long time; -#endif - pmi_msg.type = PMI_TYPE_FREQ_CHANGE; - pmi_msg.data1 = cbe_cpu_to_node(cpu); - pmi_msg.data2 = pmode; - -#ifdef DEBUG - time = jiffies; -#endif - pmi_send_message(pmi_msg); - -#ifdef DEBUG - time = jiffies - time; - time = jiffies_to_msecs(time); - pr_debug("had to wait %lu ms for a transition using " \ - "PMI\n", time); -#endif - ret = pmi_msg.data2; - pr_debug("PMI returned slow mode %d\n", ret); - - return ret; -} -EXPORT_SYMBOL_GPL(cbe_cpufreq_set_pmode_pmi); - - -static void cbe_cpufreq_handle_pmi(pmi_message_t pmi_msg) -{ - u8 node, slow_mode; - - BUG_ON(pmi_msg.type != PMI_TYPE_FREQ_CHANGE); - - node = pmi_msg.data1; - slow_mode = pmi_msg.data2; - - pmi_slow_mode_limit[node] = slow_mode; - - pr_debug("cbe_handle_pmi: node: %d max_freq: %d\n", node, slow_mode); -} - -static int pmi_notifier(struct notifier_block *nb, - unsigned long event, void *data) -{ - struct cpufreq_policy *policy = data; - struct cpufreq_frequency_table *cbe_freqs; - u8 node; - - /* Should this really be called for CPUFREQ_ADJUST, CPUFREQ_INCOMPATIBLE - * and CPUFREQ_NOTIFY policy events?) - */ - if (event == CPUFREQ_START) - return 0; - - cbe_freqs = cpufreq_frequency_get_table(policy->cpu); - node = cbe_cpu_to_node(policy->cpu); - - pr_debug("got notified, event=%lu, node=%u\n", event, node); - - if (pmi_slow_mode_limit[node] != 0) { - pr_debug("limiting node %d to slow mode %d\n", - node, pmi_slow_mode_limit[node]); - - cpufreq_verify_within_limits(policy, 0, - - cbe_freqs[pmi_slow_mode_limit[node]].frequency); - } - - return 0; -} - -static struct notifier_block pmi_notifier_block = { - .notifier_call = pmi_notifier, -}; - -static struct pmi_handler cbe_pmi_handler = { - .type = PMI_TYPE_FREQ_CHANGE, - .handle_pmi_message = cbe_cpufreq_handle_pmi, -}; - - - -static int __init cbe_cpufreq_pmi_init(void) -{ - cbe_cpufreq_has_pmi = pmi_register_handler(&cbe_pmi_handler) == 0; - - if (!cbe_cpufreq_has_pmi) - return -ENODEV; - - cpufreq_register_notifier(&pmi_notifier_block, CPUFREQ_POLICY_NOTIFIER); - - return 0; -} - -static void __exit cbe_cpufreq_pmi_exit(void) -{ - cpufreq_unregister_notifier(&pmi_notifier_block, CPUFREQ_POLICY_NOTIFIER); - pmi_unregister_handler(&cbe_pmi_handler); -} - -module_init(cbe_cpufreq_pmi_init); -module_exit(cbe_cpufreq_pmi_exit); - -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Christian Krafft <krafft@de.ibm.com>"); diff --git a/drivers/cpufreq/pxa2xx-cpufreq.c b/drivers/cpufreq/pxa2xx-cpufreq.c index fb3981ac829f..ed1ae061a687 100644 --- a/drivers/cpufreq/pxa2xx-cpufreq.c +++ b/drivers/cpufreq/pxa2xx-cpufreq.c @@ -1,20 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0-or-later /* * Copyright (C) 2002,2003 Intrinsyc Software * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - * * History: * 31-Jul-2002 : Initial version [FB] * 29-Jan-2003 : added PXA255 support [FB] @@ -26,9 +13,10 @@ * memory connected to CS0, you will need to register a platform specific * notifier which will adjust the memory access strobes to maintain a * minimum strobe width. - * */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + #include <linux/kernel.h> #include <linux/module.h> #include <linux/sched.h> @@ -36,11 +24,9 @@ #include <linux/cpufreq.h> #include <linux/err.h> #include <linux/regulator/consumer.h> +#include <linux/soc/pxa/cpu.h> #include <linux/io.h> -#include <mach/pxa2xx-regs.h> -#include <mach/smemc.h> - #ifdef DEBUG static unsigned int freq_debug; module_param(freq_debug, uint, 0); @@ -56,56 +42,40 @@ module_param(pxa27x_maxfreq, uint, 0); MODULE_PARM_DESC(pxa27x_maxfreq, "Set the pxa27x maxfreq in MHz" "(typically 624=>pxa270, 416=>pxa271, 520=>pxa272)"); -typedef struct { +struct pxa_cpufreq_data { + struct clk *clk_core; +}; +static struct pxa_cpufreq_data pxa_cpufreq_data; + +struct pxa_freqs { unsigned int khz; - unsigned int membus; - unsigned int cccr; - unsigned int div2; - unsigned int cclkcfg; int vmin; int vmax; -} pxa_freqs_t; - -/* Define the refresh period in mSec for the SDRAM and the number of rows */ -#define SDRAM_TREF 64 /* standard 64ms SDRAM */ -static unsigned int sdram_rows; - -#define CCLKCFG_TURBO 0x1 -#define CCLKCFG_FCS 0x2 -#define CCLKCFG_HALFTURBO 0x4 -#define CCLKCFG_FASTBUS 0x8 -#define MDREFR_DB2_MASK (MDREFR_K2DB2 | MDREFR_K1DB2) -#define MDREFR_DRI_MASK 0xFFF - -#define MDCNFG_DRAC2(mdcnfg) (((mdcnfg) >> 21) & 0x3) -#define MDCNFG_DRAC0(mdcnfg) (((mdcnfg) >> 5) & 0x3) +}; /* * PXA255 definitions */ -/* Use the run mode frequencies for the CPUFREQ_POLICY_PERFORMANCE policy */ -#define CCLKCFG CCLKCFG_TURBO | CCLKCFG_FCS - -static pxa_freqs_t pxa255_run_freqs[] = +static const struct pxa_freqs pxa255_run_freqs[] = { - /* CPU MEMBUS CCCR DIV2 CCLKCFG run turbo PXbus SDRAM */ - { 99500, 99500, 0x121, 1, CCLKCFG, -1, -1}, /* 99, 99, 50, 50 */ - {132700, 132700, 0x123, 1, CCLKCFG, -1, -1}, /* 133, 133, 66, 66 */ - {199100, 99500, 0x141, 0, CCLKCFG, -1, -1}, /* 199, 199, 99, 99 */ - {265400, 132700, 0x143, 1, CCLKCFG, -1, -1}, /* 265, 265, 133, 66 */ - {331800, 165900, 0x145, 1, CCLKCFG, -1, -1}, /* 331, 331, 166, 83 */ - {398100, 99500, 0x161, 0, CCLKCFG, -1, -1}, /* 398, 398, 196, 99 */ + /* CPU MEMBUS run turbo PXbus SDRAM */ + { 99500, -1, -1}, /* 99, 99, 50, 50 */ + {132700, -1, -1}, /* 133, 133, 66, 66 */ + {199100, -1, -1}, /* 199, 199, 99, 99 */ + {265400, -1, -1}, /* 265, 265, 133, 66 */ + {331800, -1, -1}, /* 331, 331, 166, 83 */ + {398100, -1, -1}, /* 398, 398, 196, 99 */ }; /* Use the turbo mode frequencies for the CPUFREQ_POLICY_POWERSAVE policy */ -static pxa_freqs_t pxa255_turbo_freqs[] = +static const struct pxa_freqs pxa255_turbo_freqs[] = { - /* CPU MEMBUS CCCR DIV2 CCLKCFG run turbo PXbus SDRAM */ - { 99500, 99500, 0x121, 1, CCLKCFG, -1, -1}, /* 99, 99, 50, 50 */ - {199100, 99500, 0x221, 0, CCLKCFG, -1, -1}, /* 99, 199, 50, 99 */ - {298500, 99500, 0x321, 0, CCLKCFG, -1, -1}, /* 99, 287, 50, 99 */ - {298600, 99500, 0x1c1, 0, CCLKCFG, -1, -1}, /* 199, 287, 99, 99 */ - {398100, 99500, 0x241, 0, CCLKCFG, -1, -1}, /* 199, 398, 99, 99 */ + /* CPU run turbo PXbus SDRAM */ + { 99500, -1, -1}, /* 99, 99, 50, 50 */ + {199100, -1, -1}, /* 99, 199, 50, 99 */ + {298500, -1, -1}, /* 99, 287, 50, 99 */ + {298600, -1, -1}, /* 199, 287, 99, 99 */ + {398100, -1, -1}, /* 199, 398, 99, 99 */ }; #define NUM_PXA25x_RUN_FREQS ARRAY_SIZE(pxa255_run_freqs) @@ -120,58 +90,23 @@ static unsigned int pxa255_turbo_table; module_param(pxa255_turbo_table, uint, 0); MODULE_PARM_DESC(pxa255_turbo_table, "Selects the frequency table (0 = run table, !0 = turbo table)"); -/* - * PXA270 definitions - * - * For the PXA27x: - * Control variables are A, L, 2N for CCCR; B, HT, T for CLKCFG. - * - * A = 0 => memory controller clock from table 3-7, - * A = 1 => memory controller clock = system bus clock - * Run mode frequency = 13 MHz * L - * Turbo mode frequency = 13 MHz * L * N - * System bus frequency = 13 MHz * L / (B + 1) - * - * In CCCR: - * A = 1 - * L = 16 oscillator to run mode ratio - * 2N = 6 2 * (turbo mode to run mode ratio) - * - * In CCLKCFG: - * B = 1 Fast bus mode - * HT = 0 Half-Turbo mode - * T = 1 Turbo mode - * - * For now, just support some of the combinations in table 3-7 of - * PXA27x Processor Family Developer's Manual to simplify frequency - * change sequences. - */ -#define PXA27x_CCCR(A, L, N2) (A << 25 | N2 << 7 | L) -#define CCLKCFG2(B, HT, T) \ - (CCLKCFG_FCS | \ - ((B) ? CCLKCFG_FASTBUS : 0) | \ - ((HT) ? CCLKCFG_HALFTURBO : 0) | \ - ((T) ? CCLKCFG_TURBO : 0)) - -static pxa_freqs_t pxa27x_freqs[] = { - {104000, 104000, PXA27x_CCCR(1, 8, 2), 0, CCLKCFG2(1, 0, 1), 900000, 1705000 }, - {156000, 104000, PXA27x_CCCR(1, 8, 3), 0, CCLKCFG2(1, 0, 1), 1000000, 1705000 }, - {208000, 208000, PXA27x_CCCR(0, 16, 2), 1, CCLKCFG2(0, 0, 1), 1180000, 1705000 }, - {312000, 208000, PXA27x_CCCR(1, 16, 3), 1, CCLKCFG2(1, 0, 1), 1250000, 1705000 }, - {416000, 208000, PXA27x_CCCR(1, 16, 4), 1, CCLKCFG2(1, 0, 1), 1350000, 1705000 }, - {520000, 208000, PXA27x_CCCR(1, 16, 5), 1, CCLKCFG2(1, 0, 1), 1450000, 1705000 }, - {624000, 208000, PXA27x_CCCR(1, 16, 6), 1, CCLKCFG2(1, 0, 1), 1550000, 1705000 } +static struct pxa_freqs pxa27x_freqs[] = { + {104000, 900000, 1705000 }, + {156000, 1000000, 1705000 }, + {208000, 1180000, 1705000 }, + {312000, 1250000, 1705000 }, + {416000, 1350000, 1705000 }, + {520000, 1450000, 1705000 }, + {624000, 1550000, 1705000 } }; #define NUM_PXA27x_FREQS ARRAY_SIZE(pxa27x_freqs) static struct cpufreq_frequency_table pxa27x_freq_table[NUM_PXA27x_FREQS+1]; -extern unsigned get_clk_frequency_khz(int info); - #ifdef CONFIG_REGULATOR -static int pxa_cpufreq_change_voltage(pxa_freqs_t *pxa_freq) +static int pxa_cpufreq_change_voltage(const struct pxa_freqs *pxa_freq) { int ret = 0; int vmin, vmax; @@ -186,32 +121,31 @@ static int pxa_cpufreq_change_voltage(pxa_freqs_t *pxa_freq) ret = regulator_set_voltage(vcc_core, vmin, vmax); if (ret) - pr_err("cpufreq: Failed to set vcc_core in [%dmV..%dmV]\n", - vmin, vmax); + pr_err("Failed to set vcc_core in [%dmV..%dmV]\n", vmin, vmax); return ret; } -static __init void pxa_cpufreq_init_voltages(void) +static void pxa_cpufreq_init_voltages(void) { vcc_core = regulator_get(NULL, "vcc_core"); if (IS_ERR(vcc_core)) { - pr_info("cpufreq: Didn't find vcc_core regulator\n"); + pr_info("Didn't find vcc_core regulator\n"); vcc_core = NULL; } else { - pr_info("cpufreq: Found vcc_core regulator\n"); + pr_info("Found vcc_core regulator\n"); } } #else -static int pxa_cpufreq_change_voltage(pxa_freqs_t *pxa_freq) +static int pxa_cpufreq_change_voltage(const struct pxa_freqs *pxa_freq) { return 0; } -static __init void pxa_cpufreq_init_voltages(void) { } +static void pxa_cpufreq_init_voltages(void) { } #endif static void find_freq_tables(struct cpufreq_frequency_table **freq_table, - pxa_freqs_t **pxa_freqs) + const struct pxa_freqs **pxa_freqs) { if (cpu_is_pxa25x()) { if (!pxa255_turbo_table) { @@ -233,154 +167,44 @@ static void pxa27x_guess_max_freq(void) { if (!pxa27x_maxfreq) { pxa27x_maxfreq = 416000; - printk(KERN_INFO "PXA CPU 27x max frequency not defined " - "(pxa27x_maxfreq), assuming pxa271 with %dkHz maxfreq\n", - pxa27x_maxfreq); + pr_info("PXA CPU 27x max frequency not defined (pxa27x_maxfreq), assuming pxa271 with %dkHz maxfreq\n", + pxa27x_maxfreq); } else { pxa27x_maxfreq *= 1000; } } -static void init_sdram_rows(void) -{ - uint32_t mdcnfg = __raw_readl(MDCNFG); - unsigned int drac2 = 0, drac0 = 0; - - if (mdcnfg & (MDCNFG_DE2 | MDCNFG_DE3)) - drac2 = MDCNFG_DRAC2(mdcnfg); - - if (mdcnfg & (MDCNFG_DE0 | MDCNFG_DE1)) - drac0 = MDCNFG_DRAC0(mdcnfg); - - sdram_rows = 1 << (11 + max(drac0, drac2)); -} - -static u32 mdrefr_dri(unsigned int freq) -{ - u32 interval = freq * SDRAM_TREF / sdram_rows; - - return (interval - (cpu_is_pxa27x() ? 31 : 0)) / 32; -} - -/* find a valid frequency point */ -static int pxa_verify_policy(struct cpufreq_policy *policy) -{ - struct cpufreq_frequency_table *pxa_freqs_table; - pxa_freqs_t *pxa_freqs; - int ret; - - find_freq_tables(&pxa_freqs_table, &pxa_freqs); - ret = cpufreq_frequency_table_verify(policy, pxa_freqs_table); - - if (freq_debug) - pr_debug("Verified CPU policy: %dKhz min to %dKhz max\n", - policy->min, policy->max); - - return ret; -} - static unsigned int pxa_cpufreq_get(unsigned int cpu) { - return get_clk_frequency_khz(0); + struct pxa_cpufreq_data *data = cpufreq_get_driver_data(); + + return (unsigned int) clk_get_rate(data->clk_core) / 1000; } -static int pxa_set_target(struct cpufreq_policy *policy, - unsigned int target_freq, - unsigned int relation) +static int pxa_set_target(struct cpufreq_policy *policy, unsigned int idx) { struct cpufreq_frequency_table *pxa_freqs_table; - pxa_freqs_t *pxa_freq_settings; - struct cpufreq_freqs freqs; - unsigned int idx; - unsigned long flags; - unsigned int new_freq_cpu, new_freq_mem; - unsigned int unused, preset_mdrefr, postset_mdrefr, cclkcfg; + const struct pxa_freqs *pxa_freq_settings; + struct pxa_cpufreq_data *data = cpufreq_get_driver_data(); + unsigned int new_freq_cpu; int ret = 0; /* Get the current policy */ find_freq_tables(&pxa_freqs_table, &pxa_freq_settings); - /* Lookup the next frequency */ - if (cpufreq_frequency_table_target(policy, pxa_freqs_table, - target_freq, relation, &idx)) { - return -EINVAL; - } - new_freq_cpu = pxa_freq_settings[idx].khz; - new_freq_mem = pxa_freq_settings[idx].membus; - freqs.old = policy->cur; - freqs.new = new_freq_cpu; if (freq_debug) - pr_debug("Changing CPU frequency to %d Mhz, (SDRAM %d Mhz)\n", - freqs.new / 1000, (pxa_freq_settings[idx].div2) ? - (new_freq_mem / 2000) : (new_freq_mem / 1000)); + pr_debug("Changing CPU frequency from %d Mhz to %d Mhz\n", + policy->cur / 1000, new_freq_cpu / 1000); - if (vcc_core && freqs.new > freqs.old) + if (vcc_core && new_freq_cpu > policy->cur) { ret = pxa_cpufreq_change_voltage(&pxa_freq_settings[idx]); - if (ret) - return ret; - /* - * Tell everyone what we're about to do... - * you should add a notify client with any platform specific - * Vcc changing capability - */ - cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE); - - /* Calculate the next MDREFR. If we're slowing down the SDRAM clock - * we need to preset the smaller DRI before the change. If we're - * speeding up we need to set the larger DRI value after the change. - */ - preset_mdrefr = postset_mdrefr = __raw_readl(MDREFR); - if ((preset_mdrefr & MDREFR_DRI_MASK) > mdrefr_dri(new_freq_mem)) { - preset_mdrefr = (preset_mdrefr & ~MDREFR_DRI_MASK); - preset_mdrefr |= mdrefr_dri(new_freq_mem); - } - postset_mdrefr = - (postset_mdrefr & ~MDREFR_DRI_MASK) | mdrefr_dri(new_freq_mem); - - /* If we're dividing the memory clock by two for the SDRAM clock, this - * must be set prior to the change. Clearing the divide must be done - * after the change. - */ - if (pxa_freq_settings[idx].div2) { - preset_mdrefr |= MDREFR_DB2_MASK; - postset_mdrefr |= MDREFR_DB2_MASK; - } else { - postset_mdrefr &= ~MDREFR_DB2_MASK; + if (ret) + return ret; } - local_irq_save(flags); - - /* Set new the CCCR and prepare CCLKCFG */ - CCCR = pxa_freq_settings[idx].cccr; - cclkcfg = pxa_freq_settings[idx].cclkcfg; - - asm volatile(" \n\ - ldr r4, [%1] /* load MDREFR */ \n\ - b 2f \n\ - .align 5 \n\ -1: \n\ - str %3, [%1] /* preset the MDREFR */ \n\ - mcr p14, 0, %2, c6, c0, 0 /* set CCLKCFG[FCS] */ \n\ - str %4, [%1] /* postset the MDREFR */ \n\ - \n\ - b 3f \n\ -2: b 1b \n\ -3: nop \n\ - " - : "=&r" (unused) - : "r" (MDREFR), "r" (cclkcfg), - "r" (preset_mdrefr), "r" (postset_mdrefr) - : "r4", "r5"); - local_irq_restore(flags); - - /* - * Tell everyone what we've just done... - * you should add a notify client with any platform specific - * SDRAM refresh timer adjustments - */ - cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); + clk_set_rate(data->clk_core, new_freq_cpu * 1000); /* * Even if voltage setting fails, we don't report it, as the frequency @@ -391,7 +215,7 @@ static int pxa_set_target(struct cpufreq_policy *policy, * bug is triggered (seems a deadlock). Should anybody find out where, * the "return 0" should become a "return ret". */ - if (vcc_core && freqs.new < freqs.old) + if (vcc_core && new_freq_cpu < policy->cur) ret = pxa_cpufreq_change_voltage(&pxa_freq_settings[idx]); return 0; @@ -402,7 +226,7 @@ static int pxa_cpufreq_init(struct cpufreq_policy *policy) int i; unsigned int freq; struct cpufreq_frequency_table *pxa255_freq_table; - pxa_freqs_t *pxa255_freqs; + const struct pxa_freqs *pxa255_freqs; /* try to guess pxa27x cpu */ if (cpu_is_pxa27x()) @@ -410,12 +234,8 @@ static int pxa_cpufreq_init(struct cpufreq_policy *policy) pxa_cpufreq_init_voltages(); - init_sdram_rows(); - /* set default policy and cpuinfo */ policy->cpuinfo.transition_latency = 1000; /* FIXME: 1 ms, assumed */ - policy->cur = get_clk_frequency_khz(0); /* current freq */ - policy->min = policy->max = policy->cur; /* Generate pxa25x the run cpufreq_frequency_table struct */ for (i = 0; i < NUM_PXA25x_RUN_FREQS; i++) { @@ -451,29 +271,38 @@ static int pxa_cpufreq_init(struct cpufreq_policy *policy) */ if (cpu_is_pxa25x()) { find_freq_tables(&pxa255_freq_table, &pxa255_freqs); - pr_info("PXA255 cpufreq using %s frequency table\n", + pr_info("using %s frequency table\n", pxa255_turbo_table ? "turbo" : "run"); - cpufreq_frequency_table_cpuinfo(policy, pxa255_freq_table); + + policy->freq_table = pxa255_freq_table; + } + else if (cpu_is_pxa27x()) { + policy->freq_table = pxa27x_freq_table; } - else if (cpu_is_pxa27x()) - cpufreq_frequency_table_cpuinfo(policy, pxa27x_freq_table); - printk(KERN_INFO "PXA CPU frequency change support initialized\n"); + pr_info("frequency change support initialized\n"); return 0; } static struct cpufreq_driver pxa_cpufreq_driver = { - .verify = pxa_verify_policy, - .target = pxa_set_target, + .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK, + .verify = cpufreq_generic_frequency_table_verify, + .target_index = pxa_set_target, .init = pxa_cpufreq_init, .get = pxa_cpufreq_get, .name = "PXA2xx", + .driver_data = &pxa_cpufreq_data, }; static int __init pxa_cpu_init(void) { int ret = -ENODEV; + + pxa_cpufreq_data.clk_core = clk_get_sys(NULL, "core"); + if (IS_ERR(pxa_cpufreq_data.clk_core)) + return PTR_ERR(pxa_cpufreq_data.clk_core); + if (cpu_is_pxa25x() || cpu_is_pxa27x()) ret = cpufreq_register_driver(&pxa_cpufreq_driver); return ret; diff --git a/drivers/cpufreq/pxa3xx-cpufreq.c b/drivers/cpufreq/pxa3xx-cpufreq.c index 9c92ef032a9e..4afa48d172db 100644 --- a/drivers/cpufreq/pxa3xx-cpufreq.c +++ b/drivers/cpufreq/pxa3xx-cpufreq.c @@ -1,10 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0-or-later /* * Copyright (C) 2008 Marvell International Ltd. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. */ #include <linux/kernel.h> @@ -12,12 +8,11 @@ #include <linux/sched.h> #include <linux/init.h> #include <linux/cpufreq.h> +#include <linux/soc/pxa/cpu.h> +#include <linux/clk/pxa.h> #include <linux/slab.h> #include <linux/io.h> -#include <mach/generic.h> -#include <mach/pxa3xx-regs.h> - #define HSS_104M (0) #define HSS_156M (1) #define HSS_208M (2) @@ -38,6 +33,28 @@ #define DMCFS_26M (0) #define DMCFS_260M (3) +#define ACCR_XPDIS (1 << 31) /* Core PLL Output Disable */ +#define ACCR_SPDIS (1 << 30) /* System PLL Output Disable */ +#define ACCR_D0CS (1 << 26) /* D0 Mode Clock Select */ +#define ACCR_PCCE (1 << 11) /* Power Mode Change Clock Enable */ +#define ACCR_DDR_D0CS (1 << 7) /* DDR SDRAM clock frequency in D0CS (PXA31x only) */ + +#define ACCR_SMCFS_MASK (0x7 << 23) /* Static Memory Controller Frequency Select */ +#define ACCR_SFLFS_MASK (0x3 << 18) /* Frequency Select for Internal Memory Controller */ +#define ACCR_XSPCLK_MASK (0x3 << 16) /* Core Frequency during Frequency Change */ +#define ACCR_HSS_MASK (0x3 << 14) /* System Bus-Clock Frequency Select */ +#define ACCR_DMCFS_MASK (0x3 << 12) /* Dynamic Memory Controller Clock Frequency Select */ +#define ACCR_XN_MASK (0x7 << 8) /* Core PLL Turbo-Mode-to-Run-Mode Ratio */ +#define ACCR_XL_MASK (0x1f) /* Core PLL Run-Mode-to-Oscillator Ratio */ + +#define ACCR_SMCFS(x) (((x) & 0x7) << 23) +#define ACCR_SFLFS(x) (((x) & 0x3) << 18) +#define ACCR_XSPCLK(x) (((x) & 0x3) << 16) +#define ACCR_HSS(x) (((x) & 0x3) << 14) +#define ACCR_DMCFS(x) (((x) & 0x3) << 12) +#define ACCR_XN(x) (((x) & 0x7) << 8) +#define ACCR_XL(x) ((x) & 0x1f) + struct pxa3xx_freq_info { unsigned int cpufreq_mhz; unsigned int core_xl : 5; @@ -93,7 +110,7 @@ static int setup_freqs_table(struct cpufreq_policy *policy, struct cpufreq_frequency_table *table; int i; - table = kzalloc((num + 1) * sizeof(*table), GFP_KERNEL); + table = kcalloc(num + 1, sizeof(*table), GFP_KERNEL); if (table == NULL) return -ENOMEM; @@ -108,51 +125,36 @@ static int setup_freqs_table(struct cpufreq_policy *policy, pxa3xx_freqs_num = num; pxa3xx_freqs_table = table; - return cpufreq_frequency_table_cpuinfo(policy, table); + policy->freq_table = table; + + return 0; } static void __update_core_freq(struct pxa3xx_freq_info *info) { - uint32_t mask = ACCR_XN_MASK | ACCR_XL_MASK; - uint32_t accr = ACCR; - uint32_t xclkcfg; - - accr &= ~(ACCR_XN_MASK | ACCR_XL_MASK | ACCR_XSPCLK_MASK); - accr |= ACCR_XN(info->core_xn) | ACCR_XL(info->core_xl); + u32 mask, disable, enable, xclkcfg; + mask = ACCR_XN_MASK | ACCR_XL_MASK; + disable = mask | ACCR_XSPCLK_MASK; + enable = ACCR_XN(info->core_xn) | ACCR_XL(info->core_xl); /* No clock until core PLL is re-locked */ - accr |= ACCR_XSPCLK(XSPCLK_NONE); - + enable |= ACCR_XSPCLK(XSPCLK_NONE); xclkcfg = (info->core_xn == 2) ? 0x3 : 0x2; /* turbo bit */ - ACCR = accr; - __asm__("mcr p14, 0, %0, c6, c0, 0\n" : : "r"(xclkcfg)); - - while ((ACSR & mask) != (accr & mask)) - cpu_relax(); + pxa3xx_clk_update_accr(disable, enable, xclkcfg, mask); } static void __update_bus_freq(struct pxa3xx_freq_info *info) { - uint32_t mask; - uint32_t accr = ACCR; - - mask = ACCR_SMCFS_MASK | ACCR_SFLFS_MASK | ACCR_HSS_MASK | - ACCR_DMCFS_MASK; + u32 mask, disable, enable; - accr &= ~mask; - accr |= ACCR_SMCFS(info->smcfs) | ACCR_SFLFS(info->sflfs) | - ACCR_HSS(info->hss) | ACCR_DMCFS(info->dmcfs); + mask = ACCR_SMCFS_MASK | ACCR_SFLFS_MASK | ACCR_HSS_MASK | + ACCR_DMCFS_MASK; + disable = mask; + enable = ACCR_SMCFS(info->smcfs) | ACCR_SFLFS(info->sflfs) | + ACCR_HSS(info->hss) | ACCR_DMCFS(info->dmcfs); - ACCR = accr; - - while ((ACSR & mask) != (accr & mask)) - cpu_relax(); -} - -static int pxa3xx_cpufreq_verify(struct cpufreq_policy *policy) -{ - return cpufreq_frequency_table_verify(policy, pxa3xx_freqs_table); + pxa3xx_clk_update_accr(disable, enable, 0, mask); } static unsigned int pxa3xx_cpufreq_get(unsigned int cpu) @@ -160,44 +162,21 @@ static unsigned int pxa3xx_cpufreq_get(unsigned int cpu) return pxa3xx_get_clk_frequency_khz(0); } -static int pxa3xx_cpufreq_set(struct cpufreq_policy *policy, - unsigned int target_freq, - unsigned int relation) +static int pxa3xx_cpufreq_set(struct cpufreq_policy *policy, unsigned int index) { struct pxa3xx_freq_info *next; - struct cpufreq_freqs freqs; unsigned long flags; - int idx; if (policy->cpu != 0) return -EINVAL; - /* Lookup the next frequency */ - if (cpufreq_frequency_table_target(policy, pxa3xx_freqs_table, - target_freq, relation, &idx)) - return -EINVAL; - - next = &pxa3xx_freqs[idx]; - - freqs.old = policy->cur; - freqs.new = next->cpufreq_mhz * 1000; - - pr_debug("CPU frequency from %d MHz to %d MHz%s\n", - freqs.old / 1000, freqs.new / 1000, - (freqs.old == freqs.new) ? " (skipped)" : ""); - - if (freqs.old == target_freq) - return 0; - - cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE); + next = &pxa3xx_freqs[index]; local_irq_save(flags); __update_core_freq(next); __update_bus_freq(next); local_irq_restore(flags); - cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); - return 0; } @@ -206,17 +185,18 @@ static int pxa3xx_cpufreq_init(struct cpufreq_policy *policy) int ret = -EINVAL; /* set default policy and cpuinfo */ - policy->cpuinfo.min_freq = 104000; - policy->cpuinfo.max_freq = (cpu_is_pxa320()) ? 806000 : 624000; + policy->min = policy->cpuinfo.min_freq = 104000; + policy->max = policy->cpuinfo.max_freq = + (cpu_is_pxa320()) ? 806000 : 624000; policy->cpuinfo.transition_latency = 1000; /* FIXME: 1 ms, assumed */ - policy->max = pxa3xx_get_clk_frequency_khz(0); - policy->cur = policy->min = policy->max; if (cpu_is_pxa300() || cpu_is_pxa310()) - ret = setup_freqs_table(policy, ARRAY_AND_SIZE(pxa300_freqs)); + ret = setup_freqs_table(policy, pxa300_freqs, + ARRAY_SIZE(pxa300_freqs)); if (cpu_is_pxa320()) - ret = setup_freqs_table(policy, ARRAY_AND_SIZE(pxa320_freqs)); + ret = setup_freqs_table(policy, pxa320_freqs, + ARRAY_SIZE(pxa320_freqs)); if (ret) { pr_err("failed to setup frequency table\n"); @@ -228,8 +208,9 @@ static int pxa3xx_cpufreq_init(struct cpufreq_policy *policy) } static struct cpufreq_driver pxa3xx_cpufreq_driver = { - .verify = pxa3xx_cpufreq_verify, - .target = pxa3xx_cpufreq_set, + .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK, + .verify = cpufreq_generic_frequency_table_verify, + .target_index = pxa3xx_cpufreq_set, .init = pxa3xx_cpufreq_init, .get = pxa3xx_cpufreq_get, .name = "pxa3xx-cpufreq", diff --git a/drivers/cpufreq/qcom-cpufreq-hw.c b/drivers/cpufreq/qcom-cpufreq-hw.c new file mode 100644 index 000000000000..8422704a3b10 --- /dev/null +++ b/drivers/cpufreq/qcom-cpufreq-hw.c @@ -0,0 +1,761 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2018, The Linux Foundation. All rights reserved. + */ + +#include <linux/bitfield.h> +#include <linux/clk-provider.h> +#include <linux/cpufreq.h> +#include <linux/init.h> +#include <linux/interconnect.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/platform_device.h> +#include <linux/pm_opp.h> +#include <linux/slab.h> +#include <linux/spinlock.h> +#include <linux/units.h> + +#define LUT_MAX_ENTRIES 40U +#define LUT_SRC GENMASK(31, 30) +#define LUT_L_VAL GENMASK(7, 0) +#define LUT_CORE_COUNT GENMASK(18, 16) +#define LUT_VOLT GENMASK(11, 0) +#define CLK_HW_DIV 2 +#define LUT_TURBO_IND 1 + +#define GT_IRQ_STATUS BIT(2) + +#define MAX_FREQ_DOMAINS 4 + +struct qcom_cpufreq_soc_data { + u32 reg_enable; + u32 reg_domain_state; + u32 reg_dcvs_ctrl; + u32 reg_freq_lut; + u32 reg_volt_lut; + u32 reg_intr_clr; + u32 reg_current_vote; + u32 reg_perf_state; + u8 lut_row_size; +}; + +struct qcom_cpufreq_data { + void __iomem *base; + + /* + * Mutex to synchronize between de-init sequence and re-starting LMh + * polling/interrupts + */ + struct mutex throttle_lock; + int throttle_irq; + char irq_name[15]; + bool cancel_throttle; + struct delayed_work throttle_work; + struct cpufreq_policy *policy; + struct clk_hw cpu_clk; + + bool per_core_dcvs; +}; + +static struct { + struct qcom_cpufreq_data *data; + const struct qcom_cpufreq_soc_data *soc_data; +} qcom_cpufreq; + +static unsigned long cpu_hw_rate, xo_rate; +static bool icc_scaling_enabled; + +static int qcom_cpufreq_set_bw(struct cpufreq_policy *policy, + unsigned long freq_khz) +{ + unsigned long freq_hz = freq_khz * 1000; + struct dev_pm_opp *opp; + struct device *dev; + int ret; + + dev = get_cpu_device(policy->cpu); + if (!dev) + return -ENODEV; + + opp = dev_pm_opp_find_freq_exact(dev, freq_hz, true); + if (IS_ERR(opp)) + return PTR_ERR(opp); + + ret = dev_pm_opp_set_opp(dev, opp); + dev_pm_opp_put(opp); + return ret; +} + +static int qcom_cpufreq_update_opp(struct device *cpu_dev, + unsigned long freq_khz, + unsigned long volt) +{ + unsigned long freq_hz = freq_khz * 1000; + int ret; + + /* Skip voltage update if the opp table is not available */ + if (!icc_scaling_enabled) + return dev_pm_opp_add(cpu_dev, freq_hz, volt); + + ret = dev_pm_opp_adjust_voltage(cpu_dev, freq_hz, volt, volt, volt); + if (ret) { + dev_err(cpu_dev, "Voltage update failed freq=%ld\n", freq_khz); + return ret; + } + + return dev_pm_opp_enable(cpu_dev, freq_hz); +} + +static int qcom_cpufreq_hw_target_index(struct cpufreq_policy *policy, + unsigned int index) +{ + struct qcom_cpufreq_data *data = policy->driver_data; + const struct qcom_cpufreq_soc_data *soc_data = qcom_cpufreq.soc_data; + unsigned long freq = policy->freq_table[index].frequency; + unsigned int i; + + writel_relaxed(index, data->base + soc_data->reg_perf_state); + + if (data->per_core_dcvs) + for (i = 1; i < cpumask_weight(policy->related_cpus); i++) + writel_relaxed(index, data->base + soc_data->reg_perf_state + i * 4); + + if (icc_scaling_enabled) + qcom_cpufreq_set_bw(policy, freq); + + return 0; +} + +static unsigned long qcom_lmh_get_throttle_freq(struct qcom_cpufreq_data *data) +{ + unsigned int lval; + + if (qcom_cpufreq.soc_data->reg_current_vote) + lval = readl_relaxed(data->base + qcom_cpufreq.soc_data->reg_current_vote) & 0x3ff; + else + lval = readl_relaxed(data->base + qcom_cpufreq.soc_data->reg_domain_state) & 0xff; + + return lval * xo_rate; +} + +/* Get the frequency requested by the cpufreq core for the CPU */ +static unsigned int qcom_cpufreq_get_freq(struct cpufreq_policy *policy) +{ + struct qcom_cpufreq_data *data; + const struct qcom_cpufreq_soc_data *soc_data; + unsigned int index; + + if (!policy) + return 0; + + data = policy->driver_data; + soc_data = qcom_cpufreq.soc_data; + + index = readl_relaxed(data->base + soc_data->reg_perf_state); + index = min(index, LUT_MAX_ENTRIES - 1); + + return policy->freq_table[index].frequency; +} + +static unsigned int __qcom_cpufreq_hw_get(struct cpufreq_policy *policy) +{ + struct qcom_cpufreq_data *data; + + if (!policy) + return 0; + + data = policy->driver_data; + + if (data->throttle_irq >= 0) + return qcom_lmh_get_throttle_freq(data) / HZ_PER_KHZ; + + return qcom_cpufreq_get_freq(policy); +} + +static unsigned int qcom_cpufreq_hw_get(unsigned int cpu) +{ + return __qcom_cpufreq_hw_get(cpufreq_cpu_get_raw(cpu)); +} + +static unsigned int qcom_cpufreq_hw_fast_switch(struct cpufreq_policy *policy, + unsigned int target_freq) +{ + struct qcom_cpufreq_data *data = policy->driver_data; + const struct qcom_cpufreq_soc_data *soc_data = qcom_cpufreq.soc_data; + unsigned int index; + unsigned int i; + + index = policy->cached_resolved_idx; + writel_relaxed(index, data->base + soc_data->reg_perf_state); + + if (data->per_core_dcvs) + for (i = 1; i < cpumask_weight(policy->related_cpus); i++) + writel_relaxed(index, data->base + soc_data->reg_perf_state + i * 4); + + return policy->freq_table[index].frequency; +} + +static int qcom_cpufreq_hw_read_lut(struct device *cpu_dev, + struct cpufreq_policy *policy) +{ + u32 data, src, lval, i, core_count, prev_freq = 0, freq; + u32 volt; + struct cpufreq_frequency_table *table; + struct dev_pm_opp *opp; + unsigned long rate; + int ret; + struct qcom_cpufreq_data *drv_data = policy->driver_data; + const struct qcom_cpufreq_soc_data *soc_data = qcom_cpufreq.soc_data; + + table = kcalloc(LUT_MAX_ENTRIES + 1, sizeof(*table), GFP_KERNEL); + if (!table) + return -ENOMEM; + + ret = dev_pm_opp_of_add_table(cpu_dev); + if (!ret) { + /* Disable all opps and cross-validate against LUT later */ + icc_scaling_enabled = true; + for (rate = 0; ; rate++) { + opp = dev_pm_opp_find_freq_ceil(cpu_dev, &rate); + if (IS_ERR(opp)) + break; + + dev_pm_opp_put(opp); + dev_pm_opp_disable(cpu_dev, rate); + } + } else if (ret != -ENODEV) { + dev_err(cpu_dev, "Invalid opp table in device tree\n"); + kfree(table); + return ret; + } else { + policy->fast_switch_possible = true; + icc_scaling_enabled = false; + } + + for (i = 0; i < LUT_MAX_ENTRIES; i++) { + data = readl_relaxed(drv_data->base + soc_data->reg_freq_lut + + i * soc_data->lut_row_size); + src = FIELD_GET(LUT_SRC, data); + lval = FIELD_GET(LUT_L_VAL, data); + core_count = FIELD_GET(LUT_CORE_COUNT, data); + + data = readl_relaxed(drv_data->base + soc_data->reg_volt_lut + + i * soc_data->lut_row_size); + volt = FIELD_GET(LUT_VOLT, data) * 1000; + + if (src) + freq = xo_rate * lval / 1000; + else + freq = cpu_hw_rate / 1000; + + if (freq != prev_freq && core_count != LUT_TURBO_IND) { + if (!qcom_cpufreq_update_opp(cpu_dev, freq, volt)) { + table[i].frequency = freq; + dev_dbg(cpu_dev, "index=%d freq=%d, core_count %d\n", i, + freq, core_count); + } else { + dev_warn(cpu_dev, "failed to update OPP for freq=%d\n", freq); + table[i].frequency = CPUFREQ_ENTRY_INVALID; + } + + } else if (core_count == LUT_TURBO_IND) { + table[i].frequency = CPUFREQ_ENTRY_INVALID; + } + + /* + * Two of the same frequencies with the same core counts means + * end of table + */ + if (i > 0 && prev_freq == freq) { + struct cpufreq_frequency_table *prev = &table[i - 1]; + + /* + * Only treat the last frequency that might be a boost + * as the boost frequency + */ + if (prev->frequency == CPUFREQ_ENTRY_INVALID) { + if (!qcom_cpufreq_update_opp(cpu_dev, prev_freq, volt)) { + prev->frequency = prev_freq; + prev->flags = CPUFREQ_BOOST_FREQ; + } else { + dev_warn(cpu_dev, "failed to update OPP for freq=%d\n", + freq); + } + } + + break; + } + + prev_freq = freq; + } + + table[i].frequency = CPUFREQ_TABLE_END; + policy->freq_table = table; + dev_pm_opp_set_sharing_cpus(cpu_dev, policy->cpus); + + return 0; +} + +static void qcom_get_related_cpus(int index, struct cpumask *m) +{ + struct device_node *cpu_np; + struct of_phandle_args args; + int cpu, ret; + + for_each_present_cpu(cpu) { + cpu_np = of_cpu_device_node_get(cpu); + if (!cpu_np) + continue; + + ret = of_parse_phandle_with_args(cpu_np, "qcom,freq-domain", + "#freq-domain-cells", 0, + &args); + of_node_put(cpu_np); + if (ret < 0) + continue; + + if (index == args.args[0]) + cpumask_set_cpu(cpu, m); + } +} + +static void qcom_lmh_dcvs_notify(struct qcom_cpufreq_data *data) +{ + struct cpufreq_policy *policy = data->policy; + int cpu = cpumask_first(policy->related_cpus); + struct device *dev = get_cpu_device(cpu); + unsigned long freq_hz, throttled_freq; + struct dev_pm_opp *opp; + + /* + * Get the h/w throttled frequency, normalize it using the + * registered opp table and use it to calculate thermal pressure. + */ + freq_hz = qcom_lmh_get_throttle_freq(data); + + opp = dev_pm_opp_find_freq_floor(dev, &freq_hz); + if (IS_ERR(opp) && PTR_ERR(opp) == -ERANGE) + opp = dev_pm_opp_find_freq_ceil(dev, &freq_hz); + + if (IS_ERR(opp)) { + dev_warn(dev, "Can't find the OPP for throttling: %pe!\n", opp); + } else { + dev_pm_opp_put(opp); + } + + throttled_freq = freq_hz / HZ_PER_KHZ; + + /* Update HW pressure (the boost frequencies are accepted) */ + arch_update_hw_pressure(policy->related_cpus, throttled_freq); + + /* + * In the unlikely case policy is unregistered do not enable + * polling or h/w interrupt + */ + mutex_lock(&data->throttle_lock); + if (data->cancel_throttle) + goto out; + + /* + * If h/w throttled frequency is higher than what cpufreq has requested + * for, then stop polling and switch back to interrupt mechanism. + */ + if (throttled_freq >= qcom_cpufreq_get_freq(cpufreq_cpu_get_raw(cpu))) + enable_irq(data->throttle_irq); + else + mod_delayed_work(system_highpri_wq, &data->throttle_work, + msecs_to_jiffies(10)); + +out: + mutex_unlock(&data->throttle_lock); +} + +static void qcom_lmh_dcvs_poll(struct work_struct *work) +{ + struct qcom_cpufreq_data *data; + + data = container_of(work, struct qcom_cpufreq_data, throttle_work.work); + qcom_lmh_dcvs_notify(data); +} + +static irqreturn_t qcom_lmh_dcvs_handle_irq(int irq, void *data) +{ + struct qcom_cpufreq_data *c_data = data; + + /* Disable interrupt and enable polling */ + disable_irq_nosync(c_data->throttle_irq); + schedule_delayed_work(&c_data->throttle_work, 0); + + if (qcom_cpufreq.soc_data->reg_intr_clr) + writel_relaxed(GT_IRQ_STATUS, + c_data->base + qcom_cpufreq.soc_data->reg_intr_clr); + + return IRQ_HANDLED; +} + +static const struct qcom_cpufreq_soc_data qcom_soc_data = { + .reg_enable = 0x0, + .reg_dcvs_ctrl = 0xbc, + .reg_freq_lut = 0x110, + .reg_volt_lut = 0x114, + .reg_current_vote = 0x704, + .reg_perf_state = 0x920, + .lut_row_size = 32, +}; + +static const struct qcom_cpufreq_soc_data epss_soc_data = { + .reg_enable = 0x0, + .reg_domain_state = 0x20, + .reg_dcvs_ctrl = 0xb0, + .reg_freq_lut = 0x100, + .reg_volt_lut = 0x200, + .reg_intr_clr = 0x308, + .reg_perf_state = 0x320, + .lut_row_size = 4, +}; + +static const struct of_device_id qcom_cpufreq_hw_match[] = { + { .compatible = "qcom,cpufreq-hw", .data = &qcom_soc_data }, + { .compatible = "qcom,cpufreq-epss", .data = &epss_soc_data }, + {} +}; +MODULE_DEVICE_TABLE(of, qcom_cpufreq_hw_match); + +static int qcom_cpufreq_hw_lmh_init(struct cpufreq_policy *policy, int index) +{ + struct qcom_cpufreq_data *data = policy->driver_data; + struct platform_device *pdev = cpufreq_get_driver_data(); + int ret; + + /* + * Look for LMh interrupt. If no interrupt line is specified / + * if there is an error, allow cpufreq to be enabled as usual. + */ + data->throttle_irq = platform_get_irq_optional(pdev, index); + if (data->throttle_irq == -ENXIO) + return 0; + if (data->throttle_irq < 0) + return data->throttle_irq; + + data->cancel_throttle = false; + + mutex_init(&data->throttle_lock); + INIT_DEFERRABLE_WORK(&data->throttle_work, qcom_lmh_dcvs_poll); + + snprintf(data->irq_name, sizeof(data->irq_name), "dcvsh-irq-%u", policy->cpu); + ret = request_threaded_irq(data->throttle_irq, NULL, qcom_lmh_dcvs_handle_irq, + IRQF_ONESHOT | IRQF_NO_AUTOEN, data->irq_name, data); + if (ret) { + dev_err(&pdev->dev, "Error registering %s: %d\n", data->irq_name, ret); + return 0; + } + + ret = irq_set_affinity_and_hint(data->throttle_irq, policy->cpus); + if (ret) + dev_err(&pdev->dev, "Failed to set CPU affinity of %s[%d]\n", + data->irq_name, data->throttle_irq); + + return 0; +} + +static int qcom_cpufreq_hw_cpu_online(struct cpufreq_policy *policy) +{ + struct qcom_cpufreq_data *data = policy->driver_data; + struct platform_device *pdev = cpufreq_get_driver_data(); + int ret; + + if (data->throttle_irq <= 0) + return 0; + + mutex_lock(&data->throttle_lock); + data->cancel_throttle = false; + mutex_unlock(&data->throttle_lock); + + ret = irq_set_affinity_and_hint(data->throttle_irq, policy->cpus); + if (ret) + dev_err(&pdev->dev, "Failed to set CPU affinity of %s[%d]\n", + data->irq_name, data->throttle_irq); + + return ret; +} + +static int qcom_cpufreq_hw_cpu_offline(struct cpufreq_policy *policy) +{ + struct qcom_cpufreq_data *data = policy->driver_data; + + if (data->throttle_irq <= 0) + return 0; + + mutex_lock(&data->throttle_lock); + data->cancel_throttle = true; + mutex_unlock(&data->throttle_lock); + + cancel_delayed_work_sync(&data->throttle_work); + irq_set_affinity_and_hint(data->throttle_irq, NULL); + disable_irq_nosync(data->throttle_irq); + + return 0; +} + +static void qcom_cpufreq_hw_lmh_exit(struct qcom_cpufreq_data *data) +{ + if (data->throttle_irq <= 0) + return; + + free_irq(data->throttle_irq, data); +} + +static int qcom_cpufreq_hw_cpu_init(struct cpufreq_policy *policy) +{ + struct platform_device *pdev = cpufreq_get_driver_data(); + struct device *dev = &pdev->dev; + struct of_phandle_args args; + struct device_node *cpu_np; + struct device *cpu_dev; + struct qcom_cpufreq_data *data; + int ret, index; + + cpu_dev = get_cpu_device(policy->cpu); + if (!cpu_dev) { + pr_err("%s: failed to get cpu%d device\n", __func__, + policy->cpu); + return -ENODEV; + } + + cpu_np = of_cpu_device_node_get(policy->cpu); + if (!cpu_np) + return -EINVAL; + + ret = of_parse_phandle_with_args(cpu_np, "qcom,freq-domain", + "#freq-domain-cells", 0, &args); + of_node_put(cpu_np); + if (ret) + return ret; + + index = args.args[0]; + data = &qcom_cpufreq.data[index]; + + /* HW should be in enabled state to proceed */ + if (!(readl_relaxed(data->base + qcom_cpufreq.soc_data->reg_enable) & 0x1)) { + dev_err(dev, "Domain-%d cpufreq hardware not enabled\n", index); + return -ENODEV; + } + + if (readl_relaxed(data->base + qcom_cpufreq.soc_data->reg_dcvs_ctrl) & 0x1) + data->per_core_dcvs = true; + + qcom_get_related_cpus(index, policy->cpus); + + policy->driver_data = data; + policy->dvfs_possible_from_any_cpu = true; + data->policy = policy; + + ret = qcom_cpufreq_hw_read_lut(cpu_dev, policy); + if (ret) { + dev_err(dev, "Domain-%d failed to read LUT\n", index); + return ret; + } + + ret = dev_pm_opp_get_opp_count(cpu_dev); + if (ret <= 0) { + dev_err(cpu_dev, "Failed to add OPPs\n"); + return -ENODEV; + } + + return qcom_cpufreq_hw_lmh_init(policy, index); +} + +static void qcom_cpufreq_hw_cpu_exit(struct cpufreq_policy *policy) +{ + struct device *cpu_dev = get_cpu_device(policy->cpu); + struct qcom_cpufreq_data *data = policy->driver_data; + + dev_pm_opp_remove_all_dynamic(cpu_dev); + dev_pm_opp_of_cpumask_remove_table(policy->related_cpus); + qcom_cpufreq_hw_lmh_exit(data); + kfree(policy->freq_table); + kfree(data); +} + +static void qcom_cpufreq_ready(struct cpufreq_policy *policy) +{ + struct qcom_cpufreq_data *data = policy->driver_data; + + if (data->throttle_irq >= 0) + enable_irq(data->throttle_irq); +} + +static struct cpufreq_driver cpufreq_qcom_hw_driver = { + .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK | + CPUFREQ_HAVE_GOVERNOR_PER_POLICY | + CPUFREQ_IS_COOLING_DEV, + .verify = cpufreq_generic_frequency_table_verify, + .target_index = qcom_cpufreq_hw_target_index, + .get = qcom_cpufreq_hw_get, + .init = qcom_cpufreq_hw_cpu_init, + .exit = qcom_cpufreq_hw_cpu_exit, + .online = qcom_cpufreq_hw_cpu_online, + .offline = qcom_cpufreq_hw_cpu_offline, + .register_em = cpufreq_register_em_with_opp, + .fast_switch = qcom_cpufreq_hw_fast_switch, + .name = "qcom-cpufreq-hw", + .ready = qcom_cpufreq_ready, + .set_boost = cpufreq_boost_set_sw, +}; + +static unsigned long qcom_cpufreq_hw_recalc_rate(struct clk_hw *hw, unsigned long parent_rate) +{ + struct qcom_cpufreq_data *data = container_of(hw, struct qcom_cpufreq_data, cpu_clk); + + return __qcom_cpufreq_hw_get(data->policy) * HZ_PER_KHZ; +} + +/* + * Since we cannot determine the closest rate of the target rate, let's just + * return the actual rate at which the clock is running at. This is needed to + * make clk_set_rate() API work properly. + */ +static int qcom_cpufreq_hw_determine_rate(struct clk_hw *hw, struct clk_rate_request *req) +{ + req->rate = qcom_cpufreq_hw_recalc_rate(hw, 0); + + return 0; +} + +static const struct clk_ops qcom_cpufreq_hw_clk_ops = { + .recalc_rate = qcom_cpufreq_hw_recalc_rate, + .determine_rate = qcom_cpufreq_hw_determine_rate, +}; + +static int qcom_cpufreq_hw_driver_probe(struct platform_device *pdev) +{ + struct clk_hw_onecell_data *clk_data; + struct device *dev = &pdev->dev; + struct device *cpu_dev; + struct clk *clk; + int ret, i, num_domains; + + clk = clk_get(dev, "xo"); + if (IS_ERR(clk)) + return PTR_ERR(clk); + + xo_rate = clk_get_rate(clk); + clk_put(clk); + + clk = clk_get(dev, "alternate"); + if (IS_ERR(clk)) + return PTR_ERR(clk); + + cpu_hw_rate = clk_get_rate(clk) / CLK_HW_DIV; + clk_put(clk); + + cpufreq_qcom_hw_driver.driver_data = pdev; + + /* Check for optional interconnect paths on CPU0 */ + cpu_dev = get_cpu_device(0); + if (!cpu_dev) + return -EPROBE_DEFER; + + ret = dev_pm_opp_of_find_icc_paths(cpu_dev, NULL); + if (ret) + return dev_err_probe(dev, ret, "Failed to find icc paths\n"); + + for (num_domains = 0; num_domains < MAX_FREQ_DOMAINS; num_domains++) + if (!platform_get_resource(pdev, IORESOURCE_MEM, num_domains)) + break; + + qcom_cpufreq.data = devm_kzalloc(dev, sizeof(struct qcom_cpufreq_data) * num_domains, + GFP_KERNEL); + if (!qcom_cpufreq.data) + return -ENOMEM; + + qcom_cpufreq.soc_data = of_device_get_match_data(dev); + if (!qcom_cpufreq.soc_data) + return -ENODEV; + + clk_data = devm_kzalloc(dev, struct_size(clk_data, hws, num_domains), GFP_KERNEL); + if (!clk_data) + return -ENOMEM; + + clk_data->num = num_domains; + + for (i = 0; i < num_domains; i++) { + struct qcom_cpufreq_data *data = &qcom_cpufreq.data[i]; + struct clk_init_data clk_init = {}; + void __iomem *base; + + base = devm_platform_ioremap_resource(pdev, i); + if (IS_ERR(base)) { + dev_err(dev, "Failed to map resource index %d\n", i); + return PTR_ERR(base); + } + + data->base = base; + + /* Register CPU clock for each frequency domain */ + clk_init.name = kasprintf(GFP_KERNEL, "qcom_cpufreq%d", i); + if (!clk_init.name) + return -ENOMEM; + + clk_init.flags = CLK_GET_RATE_NOCACHE; + clk_init.ops = &qcom_cpufreq_hw_clk_ops; + data->cpu_clk.init = &clk_init; + + ret = devm_clk_hw_register(dev, &data->cpu_clk); + if (ret < 0) { + dev_err(dev, "Failed to register clock %d: %d\n", i, ret); + kfree(clk_init.name); + return ret; + } + + clk_data->hws[i] = &data->cpu_clk; + kfree(clk_init.name); + } + + ret = devm_of_clk_add_hw_provider(dev, of_clk_hw_onecell_get, clk_data); + if (ret < 0) { + dev_err(dev, "Failed to add clock provider\n"); + return ret; + } + + ret = cpufreq_register_driver(&cpufreq_qcom_hw_driver); + if (ret) + dev_err(dev, "CPUFreq HW driver failed to register\n"); + else + dev_dbg(dev, "QCOM CPUFreq HW driver initialized\n"); + + return ret; +} + +static void qcom_cpufreq_hw_driver_remove(struct platform_device *pdev) +{ + cpufreq_unregister_driver(&cpufreq_qcom_hw_driver); +} + +static struct platform_driver qcom_cpufreq_hw_driver = { + .probe = qcom_cpufreq_hw_driver_probe, + .remove = qcom_cpufreq_hw_driver_remove, + .driver = { + .name = "qcom-cpufreq-hw", + .of_match_table = qcom_cpufreq_hw_match, + }, +}; + +static int __init qcom_cpufreq_hw_init(void) +{ + return platform_driver_register(&qcom_cpufreq_hw_driver); +} +postcore_initcall(qcom_cpufreq_hw_init); + +static void __exit qcom_cpufreq_hw_exit(void) +{ + platform_driver_unregister(&qcom_cpufreq_hw_driver); +} +module_exit(qcom_cpufreq_hw_exit); + +MODULE_DESCRIPTION("QCOM CPUFREQ HW Driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/cpufreq/qcom-cpufreq-nvmem.c b/drivers/cpufreq/qcom-cpufreq-nvmem.c new file mode 100644 index 000000000000..81e16b5a0245 --- /dev/null +++ b/drivers/cpufreq/qcom-cpufreq-nvmem.c @@ -0,0 +1,683 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2018, The Linux Foundation. All rights reserved. + */ + +/* + * In Certain QCOM SoCs like apq8096 and msm8996 that have KRYO processors, + * the CPU frequency subset and voltage value of each OPP varies + * based on the silicon variant in use. Qualcomm Process Voltage Scaling Tables + * defines the voltage and frequency value based on the msm-id in SMEM + * and speedbin blown in the efuse combination. + * The qcom-cpufreq-nvmem driver reads the msm-id and efuse value from the SoC + * to provide the OPP framework with required information. + * This is used to determine the voltage and frequency value for each OPP of + * operating-points-v2 table when it is parsed by the OPP framework. + */ + +#include <linux/cpu.h> +#include <linux/err.h> +#include <linux/init.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/nvmem-consumer.h> +#include <linux/of.h> +#include <linux/platform_device.h> +#include <linux/pm.h> +#include <linux/pm_domain.h> +#include <linux/pm_opp.h> +#include <linux/pm_runtime.h> +#include <linux/slab.h> +#include <linux/soc/qcom/smem.h> + +#include <dt-bindings/arm/qcom,ids.h> + +enum ipq806x_versions { + IPQ8062_VERSION = 0, + IPQ8064_VERSION, + IPQ8065_VERSION, +}; + +#define IPQ6000_VERSION BIT(2) + +enum ipq8074_versions { + IPQ8074_HAWKEYE_VERSION = 0, + IPQ8074_ACORN_VERSION, +}; + +struct qcom_cpufreq_drv; + +struct qcom_cpufreq_match_data { + int (*get_version)(struct device *cpu_dev, + struct nvmem_cell *speedbin_nvmem, + char **pvs_name, + struct qcom_cpufreq_drv *drv); + const char **pd_names; + unsigned int num_pd_names; +}; + +struct qcom_cpufreq_drv_cpu { + int opp_token; + struct dev_pm_domain_list *pd_list; +}; + +struct qcom_cpufreq_drv { + u32 versions; + const struct qcom_cpufreq_match_data *data; + struct qcom_cpufreq_drv_cpu cpus[]; +}; + +static struct platform_device *cpufreq_dt_pdev, *cpufreq_pdev; + +static int qcom_cpufreq_simple_get_version(struct device *cpu_dev, + struct nvmem_cell *speedbin_nvmem, + char **pvs_name, + struct qcom_cpufreq_drv *drv) +{ + u8 *speedbin; + + *pvs_name = NULL; + speedbin = nvmem_cell_read(speedbin_nvmem, NULL); + if (IS_ERR(speedbin)) + return PTR_ERR(speedbin); + + dev_dbg(cpu_dev, "speedbin: %d\n", *speedbin); + drv->versions = 1 << *speedbin; + kfree(speedbin); + return 0; +} + +static void get_krait_bin_format_a(struct device *cpu_dev, + int *speed, int *pvs, + u8 *buf) +{ + u32 pte_efuse; + + pte_efuse = *((u32 *)buf); + + *speed = pte_efuse & 0xf; + if (*speed == 0xf) + *speed = (pte_efuse >> 4) & 0xf; + + if (*speed == 0xf) { + *speed = 0; + dev_warn(cpu_dev, "Speed bin: Defaulting to %d\n", *speed); + } else { + dev_dbg(cpu_dev, "Speed bin: %d\n", *speed); + } + + *pvs = (pte_efuse >> 10) & 0x7; + if (*pvs == 0x7) + *pvs = (pte_efuse >> 13) & 0x7; + + if (*pvs == 0x7) { + *pvs = 0; + dev_warn(cpu_dev, "PVS bin: Defaulting to %d\n", *pvs); + } else { + dev_dbg(cpu_dev, "PVS bin: %d\n", *pvs); + } +} + +static void get_krait_bin_format_b(struct device *cpu_dev, + int *speed, int *pvs, int *pvs_ver, + u8 *buf) +{ + u32 pte_efuse, redundant_sel; + + pte_efuse = *((u32 *)buf); + redundant_sel = (pte_efuse >> 24) & 0x7; + + *pvs_ver = (pte_efuse >> 4) & 0x3; + + switch (redundant_sel) { + case 1: + *pvs = ((pte_efuse >> 28) & 0x8) | ((pte_efuse >> 6) & 0x7); + *speed = (pte_efuse >> 27) & 0xf; + break; + case 2: + *pvs = (pte_efuse >> 27) & 0xf; + *speed = pte_efuse & 0x7; + break; + default: + /* 4 bits of PVS are in efuse register bits 31, 8-6. */ + *pvs = ((pte_efuse >> 28) & 0x8) | ((pte_efuse >> 6) & 0x7); + *speed = pte_efuse & 0x7; + } + + /* Check SPEED_BIN_BLOW_STATUS */ + if (pte_efuse & BIT(3)) { + dev_dbg(cpu_dev, "Speed bin: %d\n", *speed); + } else { + dev_warn(cpu_dev, "Speed bin not set. Defaulting to 0!\n"); + *speed = 0; + } + + /* Check PVS_BLOW_STATUS */ + pte_efuse = *(((u32 *)buf) + 1); + pte_efuse &= BIT(21); + if (pte_efuse) { + dev_dbg(cpu_dev, "PVS bin: %d\n", *pvs); + } else { + dev_warn(cpu_dev, "PVS bin not set. Defaulting to 0!\n"); + *pvs = 0; + } + + dev_dbg(cpu_dev, "PVS version: %d\n", *pvs_ver); +} + +static int qcom_cpufreq_kryo_name_version(struct device *cpu_dev, + struct nvmem_cell *speedbin_nvmem, + char **pvs_name, + struct qcom_cpufreq_drv *drv) +{ + size_t len; + u32 msm_id; + u8 *speedbin; + int ret; + *pvs_name = NULL; + + ret = qcom_smem_get_soc_id(&msm_id); + if (ret) + return ret; + + speedbin = nvmem_cell_read(speedbin_nvmem, &len); + if (IS_ERR(speedbin)) + return PTR_ERR(speedbin); + + switch (msm_id) { + case QCOM_ID_MSM8996: + case QCOM_ID_APQ8096: + case QCOM_ID_IPQ5332: + case QCOM_ID_IPQ5322: + case QCOM_ID_IPQ5312: + case QCOM_ID_IPQ5302: + case QCOM_ID_IPQ5300: + case QCOM_ID_IPQ5321: + case QCOM_ID_IPQ9514: + case QCOM_ID_IPQ9550: + case QCOM_ID_IPQ9554: + case QCOM_ID_IPQ9570: + case QCOM_ID_IPQ9574: + drv->versions = 1 << (unsigned int)(*speedbin); + break; + case QCOM_ID_IPQ5424: + case QCOM_ID_IPQ5404: + drv->versions = (*speedbin == 0x3b) ? BIT(1) : BIT(0); + break; + case QCOM_ID_MSM8996SG: + case QCOM_ID_APQ8096SG: + drv->versions = 1 << ((unsigned int)(*speedbin) + 4); + break; + default: + BUG(); + break; + } + + kfree(speedbin); + return 0; +} + +static int qcom_cpufreq_krait_name_version(struct device *cpu_dev, + struct nvmem_cell *speedbin_nvmem, + char **pvs_name, + struct qcom_cpufreq_drv *drv) +{ + int speed = 0, pvs = 0, pvs_ver = 0; + u8 *speedbin; + size_t len; + int ret = 0; + + speedbin = nvmem_cell_read(speedbin_nvmem, &len); + + if (IS_ERR(speedbin)) + return PTR_ERR(speedbin); + + switch (len) { + case 4: + get_krait_bin_format_a(cpu_dev, &speed, &pvs, speedbin); + break; + case 8: + get_krait_bin_format_b(cpu_dev, &speed, &pvs, &pvs_ver, + speedbin); + break; + default: + dev_err(cpu_dev, "Unable to read nvmem data. Defaulting to 0!\n"); + ret = -ENODEV; + goto len_error; + } + + snprintf(*pvs_name, sizeof("speedXX-pvsXX-vXX"), "speed%d-pvs%d-v%d", + speed, pvs, pvs_ver); + + drv->versions = (1 << speed); + +len_error: + kfree(speedbin); + return ret; +} + +static const struct of_device_id qcom_cpufreq_ipq806x_match_list[] __maybe_unused = { + { .compatible = "qcom,ipq8062", .data = (const void *)QCOM_ID_IPQ8062 }, + { .compatible = "qcom,ipq8064", .data = (const void *)QCOM_ID_IPQ8064 }, + { .compatible = "qcom,ipq8065", .data = (const void *)QCOM_ID_IPQ8065 }, + { .compatible = "qcom,ipq8066", .data = (const void *)QCOM_ID_IPQ8066 }, + { .compatible = "qcom,ipq8068", .data = (const void *)QCOM_ID_IPQ8068 }, + { .compatible = "qcom,ipq8069", .data = (const void *)QCOM_ID_IPQ8069 }, +}; + +static int qcom_cpufreq_ipq8064_name_version(struct device *cpu_dev, + struct nvmem_cell *speedbin_nvmem, + char **pvs_name, + struct qcom_cpufreq_drv *drv) +{ + int msm_id = -1, ret = 0; + int speed = 0, pvs = 0; + u8 *speedbin; + size_t len; + + speedbin = nvmem_cell_read(speedbin_nvmem, &len); + if (IS_ERR(speedbin)) + return PTR_ERR(speedbin); + + if (len != 4) { + dev_err(cpu_dev, "Unable to read nvmem data. Defaulting to 0!\n"); + ret = -ENODEV; + goto exit; + } + + get_krait_bin_format_a(cpu_dev, &speed, &pvs, speedbin); + + ret = qcom_smem_get_soc_id(&msm_id); + if (ret == -ENODEV) { + const struct of_device_id *match; + struct device_node *root; + + root = of_find_node_by_path("/"); + if (!root) { + ret = -ENODEV; + goto exit; + } + + /* Fallback to compatible match with no SMEM initialized */ + match = of_match_node(qcom_cpufreq_ipq806x_match_list, root); + of_node_put(root); + if (!match) { + ret = -ENODEV; + goto exit; + } + + /* We found a matching device, get the msm_id from the data entry */ + msm_id = (int)(uintptr_t)match->data; + ret = 0; + } else if (ret) { + goto exit; + } + + switch (msm_id) { + case QCOM_ID_IPQ8062: + drv->versions = BIT(IPQ8062_VERSION); + break; + case QCOM_ID_IPQ8064: + case QCOM_ID_IPQ8066: + case QCOM_ID_IPQ8068: + drv->versions = BIT(IPQ8064_VERSION); + break; + case QCOM_ID_IPQ8065: + case QCOM_ID_IPQ8069: + drv->versions = BIT(IPQ8065_VERSION); + break; + default: + dev_err(cpu_dev, + "SoC ID %u is not part of IPQ8064 family, limiting to 1.0GHz!\n", + msm_id); + drv->versions = BIT(IPQ8062_VERSION); + break; + } + + /* IPQ8064 speed is never fused. Only pvs values are fused. */ + snprintf(*pvs_name, sizeof("speed0-pvsXX"), "speed0-pvs%d", pvs); + +exit: + kfree(speedbin); + return ret; +} + +static int qcom_cpufreq_ipq6018_name_version(struct device *cpu_dev, + struct nvmem_cell *speedbin_nvmem, + char **pvs_name, + struct qcom_cpufreq_drv *drv) +{ + u32 msm_id; + int ret; + u8 *speedbin; + *pvs_name = NULL; + + ret = qcom_smem_get_soc_id(&msm_id); + if (ret) + return ret; + + speedbin = nvmem_cell_read(speedbin_nvmem, NULL); + if (IS_ERR(speedbin)) + return PTR_ERR(speedbin); + + switch (msm_id) { + case QCOM_ID_IPQ6005: + case QCOM_ID_IPQ6010: + case QCOM_ID_IPQ6018: + case QCOM_ID_IPQ6028: + /* Fuse Value Freq BIT to set + * --------------------------------- + * 2’b0 No Limit BIT(0) + * 2’b1 1.5 GHz BIT(1) + */ + drv->versions = 1 << (unsigned int)(*speedbin); + break; + case QCOM_ID_IPQ6000: + /* + * IPQ6018 family only has one bit to advertise the CPU + * speed-bin, but that is not enough for IPQ6000 which + * is only rated up to 1.2GHz. + * So for IPQ6000 manually set BIT(2) based on SMEM ID. + */ + drv->versions = IPQ6000_VERSION; + break; + default: + dev_err(cpu_dev, + "SoC ID %u is not part of IPQ6018 family, limiting to 1.2GHz!\n", + msm_id); + drv->versions = IPQ6000_VERSION; + break; + } + + kfree(speedbin); + return 0; +} + +static int qcom_cpufreq_ipq8074_name_version(struct device *cpu_dev, + struct nvmem_cell *speedbin_nvmem, + char **pvs_name, + struct qcom_cpufreq_drv *drv) +{ + u32 msm_id; + int ret; + *pvs_name = NULL; + + ret = qcom_smem_get_soc_id(&msm_id); + if (ret) + return ret; + + switch (msm_id) { + case QCOM_ID_IPQ8070A: + case QCOM_ID_IPQ8071A: + case QCOM_ID_IPQ8172: + case QCOM_ID_IPQ8173: + case QCOM_ID_IPQ8174: + drv->versions = BIT(IPQ8074_ACORN_VERSION); + break; + case QCOM_ID_IPQ8072A: + case QCOM_ID_IPQ8074A: + case QCOM_ID_IPQ8076A: + case QCOM_ID_IPQ8078A: + drv->versions = BIT(IPQ8074_HAWKEYE_VERSION); + break; + default: + dev_err(cpu_dev, + "SoC ID %u is not part of IPQ8074 family, limiting to 1.4GHz!\n", + msm_id); + drv->versions = BIT(IPQ8074_ACORN_VERSION); + break; + } + + return 0; +} + +static const struct qcom_cpufreq_match_data match_data_kryo = { + .get_version = qcom_cpufreq_kryo_name_version, +}; + +static const struct qcom_cpufreq_match_data match_data_krait = { + .get_version = qcom_cpufreq_krait_name_version, +}; + +static const struct qcom_cpufreq_match_data match_data_msm8909 = { + .get_version = qcom_cpufreq_simple_get_version, + .pd_names = (const char *[]) { "perf" }, + .num_pd_names = 1, +}; + +static const struct qcom_cpufreq_match_data match_data_qcs404 = { + .pd_names = (const char *[]) { "cpr" }, + .num_pd_names = 1, +}; + +static const struct qcom_cpufreq_match_data match_data_ipq6018 = { + .get_version = qcom_cpufreq_ipq6018_name_version, +}; + +static const struct qcom_cpufreq_match_data match_data_ipq8064 = { + .get_version = qcom_cpufreq_ipq8064_name_version, +}; + +static const struct qcom_cpufreq_match_data match_data_ipq8074 = { + .get_version = qcom_cpufreq_ipq8074_name_version, +}; + +static void qcom_cpufreq_suspend_pd_devs(struct qcom_cpufreq_drv *drv, unsigned int cpu) +{ + struct dev_pm_domain_list *pd_list = drv->cpus[cpu].pd_list; + int i; + + if (!pd_list) + return; + + for (i = 0; i < pd_list->num_pds; i++) + device_set_awake_path(pd_list->pd_devs[i]); +} + +static int qcom_cpufreq_probe(struct platform_device *pdev) +{ + struct qcom_cpufreq_drv *drv; + struct nvmem_cell *speedbin_nvmem; + struct device *cpu_dev; + char pvs_name_buffer[] = "speedXX-pvsXX-vXX"; + char *pvs_name = pvs_name_buffer; + unsigned cpu; + const struct of_device_id *match; + int ret; + + cpu_dev = get_cpu_device(0); + if (!cpu_dev) + return -ENODEV; + + struct device_node *np __free(device_node) = + dev_pm_opp_of_get_opp_desc_node(cpu_dev); + if (!np) + return -ENOENT; + + ret = of_device_is_compatible(np, "operating-points-v2-kryo-cpu") || + of_device_is_compatible(np, "operating-points-v2-krait-cpu"); + if (!ret) + return -ENOENT; + + drv = devm_kzalloc(&pdev->dev, struct_size(drv, cpus, num_possible_cpus()), + GFP_KERNEL); + if (!drv) + return -ENOMEM; + + match = pdev->dev.platform_data; + drv->data = match->data; + if (!drv->data) + return -ENODEV; + + if (drv->data->get_version) { + speedbin_nvmem = of_nvmem_cell_get(np, NULL); + if (IS_ERR(speedbin_nvmem)) + return dev_err_probe(cpu_dev, PTR_ERR(speedbin_nvmem), + "Could not get nvmem cell\n"); + + ret = drv->data->get_version(cpu_dev, + speedbin_nvmem, &pvs_name, drv); + if (ret) { + nvmem_cell_put(speedbin_nvmem); + return ret; + } + nvmem_cell_put(speedbin_nvmem); + } + + for_each_present_cpu(cpu) { + struct dev_pm_opp_config config = { + .supported_hw = NULL, + }; + + cpu_dev = get_cpu_device(cpu); + if (NULL == cpu_dev) { + ret = -ENODEV; + goto free_opp; + } + + if (drv->data->get_version) { + config.supported_hw = &drv->versions; + config.supported_hw_count = 1; + + if (pvs_name) + config.prop_name = pvs_name; + } + + if (config.supported_hw) { + drv->cpus[cpu].opp_token = dev_pm_opp_set_config(cpu_dev, &config); + if (drv->cpus[cpu].opp_token < 0) { + ret = drv->cpus[cpu].opp_token; + dev_err(cpu_dev, "Failed to set OPP config\n"); + goto free_opp; + } + } + + if (drv->data->pd_names) { + struct dev_pm_domain_attach_data attach_data = { + .pd_names = drv->data->pd_names, + .num_pd_names = drv->data->num_pd_names, + .pd_flags = PD_FLAG_DEV_LINK_ON | + PD_FLAG_REQUIRED_OPP, + }; + + ret = dev_pm_domain_attach_list(cpu_dev, &attach_data, + &drv->cpus[cpu].pd_list); + if (ret < 0) + goto free_opp; + } + } + + cpufreq_dt_pdev = platform_device_register_simple("cpufreq-dt", -1, + NULL, 0); + if (!IS_ERR(cpufreq_dt_pdev)) { + platform_set_drvdata(pdev, drv); + return 0; + } + + ret = PTR_ERR(cpufreq_dt_pdev); + dev_err(cpu_dev, "Failed to register platform device\n"); + +free_opp: + for_each_present_cpu(cpu) { + dev_pm_domain_detach_list(drv->cpus[cpu].pd_list); + dev_pm_opp_clear_config(drv->cpus[cpu].opp_token); + } + return ret; +} + +static void qcom_cpufreq_remove(struct platform_device *pdev) +{ + struct qcom_cpufreq_drv *drv = platform_get_drvdata(pdev); + unsigned int cpu; + + platform_device_unregister(cpufreq_dt_pdev); + + for_each_present_cpu(cpu) { + dev_pm_domain_detach_list(drv->cpus[cpu].pd_list); + dev_pm_opp_clear_config(drv->cpus[cpu].opp_token); + } +} + +static int qcom_cpufreq_suspend(struct device *dev) +{ + struct qcom_cpufreq_drv *drv = dev_get_drvdata(dev); + unsigned int cpu; + + for_each_present_cpu(cpu) + qcom_cpufreq_suspend_pd_devs(drv, cpu); + + return 0; +} + +static DEFINE_SIMPLE_DEV_PM_OPS(qcom_cpufreq_pm_ops, qcom_cpufreq_suspend, NULL); + +static struct platform_driver qcom_cpufreq_driver = { + .probe = qcom_cpufreq_probe, + .remove = qcom_cpufreq_remove, + .driver = { + .name = "qcom-cpufreq-nvmem", + .pm = pm_sleep_ptr(&qcom_cpufreq_pm_ops), + }, +}; + +static const struct of_device_id qcom_cpufreq_match_list[] __initconst __maybe_unused = { + { .compatible = "qcom,apq8096", .data = &match_data_kryo }, + { .compatible = "qcom,msm8909", .data = &match_data_msm8909 }, + { .compatible = "qcom,msm8996", .data = &match_data_kryo }, + { .compatible = "qcom,qcs404", .data = &match_data_qcs404 }, + { .compatible = "qcom,ipq5332", .data = &match_data_kryo }, + { .compatible = "qcom,ipq5424", .data = &match_data_kryo }, + { .compatible = "qcom,ipq6018", .data = &match_data_ipq6018 }, + { .compatible = "qcom,ipq8064", .data = &match_data_ipq8064 }, + { .compatible = "qcom,ipq8074", .data = &match_data_ipq8074 }, + { .compatible = "qcom,apq8064", .data = &match_data_krait }, + { .compatible = "qcom,ipq9574", .data = &match_data_kryo }, + { .compatible = "qcom,msm8974", .data = &match_data_krait }, + { .compatible = "qcom,msm8960", .data = &match_data_krait }, + {}, +}; +MODULE_DEVICE_TABLE(of, qcom_cpufreq_match_list); + +/* + * Since the driver depends on smem and nvmem drivers, which may + * return EPROBE_DEFER, all the real activity is done in the probe, + * which may be defered as well. The init here is only registering + * the driver and the platform device. + */ +static int __init qcom_cpufreq_init(void) +{ + struct device_node *np __free(device_node) = of_find_node_by_path("/"); + const struct of_device_id *match; + int ret; + + if (!np) + return -ENODEV; + + match = of_match_node(qcom_cpufreq_match_list, np); + if (!match) + return -ENODEV; + + ret = platform_driver_register(&qcom_cpufreq_driver); + if (unlikely(ret < 0)) + return ret; + + cpufreq_pdev = platform_device_register_data(NULL, "qcom-cpufreq-nvmem", + -1, match, sizeof(*match)); + ret = PTR_ERR_OR_ZERO(cpufreq_pdev); + if (0 == ret) + return 0; + + platform_driver_unregister(&qcom_cpufreq_driver); + return ret; +} +module_init(qcom_cpufreq_init); + +static void __exit qcom_cpufreq_exit(void) +{ + platform_device_unregister(cpufreq_pdev); + platform_driver_unregister(&qcom_cpufreq_driver); +} +module_exit(qcom_cpufreq_exit); + +MODULE_DESCRIPTION("Qualcomm Technologies, Inc. CPUfreq driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/cpufreq/qoriq-cpufreq.c b/drivers/cpufreq/qoriq-cpufreq.c new file mode 100644 index 000000000000..8d1f5ac59132 --- /dev/null +++ b/drivers/cpufreq/qoriq-cpufreq.c @@ -0,0 +1,305 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2013 Freescale Semiconductor, Inc. + * + * CPU Frequency Scaling driver for Freescale QorIQ SoCs. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/clk.h> +#include <linux/clk-provider.h> +#include <linux/cpufreq.h> +#include <linux/errno.h> +#include <linux/init.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/mutex.h> +#include <linux/of.h> +#include <linux/slab.h> +#include <linux/smp.h> +#include <linux/platform_device.h> + +/** + * struct cpu_data + * @pclk: the parent clock of cpu + * @table: frequency table + */ +struct cpu_data { + struct clk **pclk; + struct cpufreq_frequency_table *table; +}; + +/** + * struct soc_data - SoC specific data + * @flags: SOC_xxx + */ +struct soc_data { + u32 flags; +}; + +static u32 get_bus_freq(void) +{ + struct device_node *soc; + u32 sysfreq; + struct clk *pltclk; + int ret; + + /* get platform freq by searching bus-frequency property */ + soc = of_find_node_by_type(NULL, "soc"); + if (soc) { + ret = of_property_read_u32(soc, "bus-frequency", &sysfreq); + of_node_put(soc); + if (!ret) + return sysfreq; + } + + /* get platform freq by its clock name */ + pltclk = clk_get(NULL, "cg-pll0-div1"); + if (IS_ERR(pltclk)) { + pr_err("%s: can't get bus frequency %ld\n", + __func__, PTR_ERR(pltclk)); + return PTR_ERR(pltclk); + } + + return clk_get_rate(pltclk); +} + +static struct clk *cpu_to_clk(int cpu) +{ + struct device_node *np; + struct clk *clk; + + if (!cpu_present(cpu)) + return NULL; + + np = of_get_cpu_node(cpu, NULL); + if (!np) + return NULL; + + clk = of_clk_get(np, 0); + of_node_put(np); + return clk; +} + +/* traverse cpu nodes to get cpu mask of sharing clock wire */ +static void set_affected_cpus(struct cpufreq_policy *policy) +{ + struct cpumask *dstp = policy->cpus; + struct clk *clk; + int i; + + for_each_present_cpu(i) { + clk = cpu_to_clk(i); + if (IS_ERR(clk)) { + pr_err("%s: no clock for cpu %d\n", __func__, i); + continue; + } + + if (clk_is_match(policy->clk, clk)) + cpumask_set_cpu(i, dstp); + } +} + +/* reduce the duplicated frequencies in frequency table */ +static void freq_table_redup(struct cpufreq_frequency_table *freq_table, + int count) +{ + int i, j; + + for (i = 1; i < count; i++) { + for (j = 0; j < i; j++) { + if (freq_table[j].frequency == CPUFREQ_ENTRY_INVALID || + freq_table[j].frequency != + freq_table[i].frequency) + continue; + + freq_table[i].frequency = CPUFREQ_ENTRY_INVALID; + break; + } + } +} + +/* sort the frequencies in frequency table in descenting order */ +static void freq_table_sort(struct cpufreq_frequency_table *freq_table, + int count) +{ + int i, j, ind; + unsigned int freq, max_freq; + struct cpufreq_frequency_table table; + + for (i = 0; i < count - 1; i++) { + max_freq = freq_table[i].frequency; + ind = i; + for (j = i + 1; j < count; j++) { + freq = freq_table[j].frequency; + if (freq == CPUFREQ_ENTRY_INVALID || + freq <= max_freq) + continue; + ind = j; + max_freq = freq; + } + + if (ind != i) { + /* exchange the frequencies */ + table.driver_data = freq_table[i].driver_data; + table.frequency = freq_table[i].frequency; + freq_table[i].driver_data = freq_table[ind].driver_data; + freq_table[i].frequency = freq_table[ind].frequency; + freq_table[ind].driver_data = table.driver_data; + freq_table[ind].frequency = table.frequency; + } + } +} + +static int qoriq_cpufreq_cpu_init(struct cpufreq_policy *policy) +{ + struct device_node *np; + int i, count; + u32 freq; + struct clk *clk; + const struct clk_hw *hwclk; + struct cpufreq_frequency_table *table; + struct cpu_data *data; + unsigned int cpu = policy->cpu; + u64 u64temp; + + np = of_get_cpu_node(cpu, NULL); + if (!np) + return -ENODEV; + + data = kzalloc(sizeof(*data), GFP_KERNEL); + if (!data) + goto err_np; + + policy->clk = of_clk_get(np, 0); + if (IS_ERR(policy->clk)) { + pr_err("%s: no clock information\n", __func__); + goto err_nomem2; + } + + hwclk = __clk_get_hw(policy->clk); + count = clk_hw_get_num_parents(hwclk); + + data->pclk = kcalloc(count, sizeof(struct clk *), GFP_KERNEL); + if (!data->pclk) + goto err_nomem2; + + table = kcalloc(count + 1, sizeof(*table), GFP_KERNEL); + if (!table) + goto err_pclk; + + for (i = 0; i < count; i++) { + clk = clk_hw_get_parent_by_index(hwclk, i)->clk; + data->pclk[i] = clk; + freq = clk_get_rate(clk); + table[i].frequency = freq / 1000; + table[i].driver_data = i; + } + freq_table_redup(table, count); + freq_table_sort(table, count); + table[i].frequency = CPUFREQ_TABLE_END; + policy->freq_table = table; + data->table = table; + + /* update ->cpus if we have cluster, no harm if not */ + set_affected_cpus(policy); + policy->driver_data = data; + + /* Minimum transition latency is 12 platform clocks */ + u64temp = 12ULL * NSEC_PER_SEC; + do_div(u64temp, get_bus_freq()); + policy->cpuinfo.transition_latency = u64temp + 1; + + of_node_put(np); + + return 0; + +err_pclk: + kfree(data->pclk); +err_nomem2: + kfree(data); +err_np: + of_node_put(np); + + return -ENODEV; +} + +static void qoriq_cpufreq_cpu_exit(struct cpufreq_policy *policy) +{ + struct cpu_data *data = policy->driver_data; + + kfree(data->pclk); + kfree(data->table); + kfree(data); + policy->driver_data = NULL; +} + +static int qoriq_cpufreq_target(struct cpufreq_policy *policy, + unsigned int index) +{ + struct clk *parent; + struct cpu_data *data = policy->driver_data; + + parent = data->pclk[data->table[index].driver_data]; + return clk_set_parent(policy->clk, parent); +} + +static struct cpufreq_driver qoriq_cpufreq_driver = { + .name = "qoriq_cpufreq", + .flags = CPUFREQ_CONST_LOOPS | + CPUFREQ_IS_COOLING_DEV, + .init = qoriq_cpufreq_cpu_init, + .exit = qoriq_cpufreq_cpu_exit, + .verify = cpufreq_generic_frequency_table_verify, + .target_index = qoriq_cpufreq_target, + .get = cpufreq_generic_get, +}; + +static const struct of_device_id qoriq_cpufreq_blacklist[] = { + /* e6500 cannot use cpufreq due to erratum A-008083 */ + { .compatible = "fsl,b4420-clockgen", }, + { .compatible = "fsl,b4860-clockgen", }, + { .compatible = "fsl,t2080-clockgen", }, + { .compatible = "fsl,t4240-clockgen", }, + {} +}; + +static int qoriq_cpufreq_probe(struct platform_device *pdev) +{ + int ret; + struct device_node *np; + + np = of_find_matching_node(NULL, qoriq_cpufreq_blacklist); + if (np) { + of_node_put(np); + dev_info(&pdev->dev, "Disabling due to erratum A-008083"); + return -ENODEV; + } + + ret = cpufreq_register_driver(&qoriq_cpufreq_driver); + if (ret) + return ret; + + dev_info(&pdev->dev, "Freescale QorIQ CPU frequency scaling driver\n"); + return 0; +} + +static void qoriq_cpufreq_remove(struct platform_device *pdev) +{ + cpufreq_unregister_driver(&qoriq_cpufreq_driver); +} + +static struct platform_driver qoriq_cpufreq_platform_driver = { + .driver = { + .name = "qoriq-cpufreq", + }, + .probe = qoriq_cpufreq_probe, + .remove = qoriq_cpufreq_remove, +}; +module_platform_driver(qoriq_cpufreq_platform_driver); + +MODULE_ALIAS("platform:qoriq-cpufreq"); +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Tang Yuantian <Yuantian.Tang@freescale.com>"); +MODULE_DESCRIPTION("cpufreq driver for Freescale QorIQ series SoCs"); diff --git a/drivers/cpufreq/raspberrypi-cpufreq.c b/drivers/cpufreq/raspberrypi-cpufreq.c new file mode 100644 index 000000000000..5050932954e3 --- /dev/null +++ b/drivers/cpufreq/raspberrypi-cpufreq.c @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Raspberry Pi cpufreq driver + * + * Copyright (C) 2019, Nicolas Saenz Julienne <nsaenzjulienne@suse.de> + */ + +#include <linux/clk.h> +#include <linux/cpu.h> +#include <linux/cpufreq.h> +#include <linux/module.h> +#include <linux/platform_device.h> +#include <linux/pm_opp.h> + +#define RASPBERRYPI_FREQ_INTERVAL 100000000 + +static struct platform_device *cpufreq_dt; + +static int raspberrypi_cpufreq_probe(struct platform_device *pdev) +{ + struct device *cpu_dev; + unsigned long min, max; + unsigned long rate; + struct clk *clk; + int ret; + + cpu_dev = get_cpu_device(0); + if (!cpu_dev) { + pr_err("Cannot get CPU for cpufreq driver\n"); + return -ENODEV; + } + + clk = clk_get(cpu_dev, NULL); + if (IS_ERR(clk)) { + dev_err(cpu_dev, "Cannot get clock for CPU0\n"); + return PTR_ERR(clk); + } + + /* + * The max and min frequencies are configurable in the Raspberry Pi + * firmware, so we query them at runtime. + */ + min = roundup(clk_round_rate(clk, 0), RASPBERRYPI_FREQ_INTERVAL); + max = roundup(clk_round_rate(clk, ULONG_MAX), RASPBERRYPI_FREQ_INTERVAL); + clk_put(clk); + + for (rate = min; rate <= max; rate += RASPBERRYPI_FREQ_INTERVAL) { + ret = dev_pm_opp_add(cpu_dev, rate, 0); + if (ret) + goto remove_opp; + } + + cpufreq_dt = platform_device_register_simple("cpufreq-dt", -1, NULL, 0); + ret = PTR_ERR_OR_ZERO(cpufreq_dt); + if (ret) { + dev_err(cpu_dev, "Failed to create platform device, %d\n", ret); + goto remove_opp; + } + + return 0; + +remove_opp: + dev_pm_opp_remove_all_dynamic(cpu_dev); + + return ret; +} + +static void raspberrypi_cpufreq_remove(struct platform_device *pdev) +{ + struct device *cpu_dev; + + cpu_dev = get_cpu_device(0); + if (cpu_dev) + dev_pm_opp_remove_all_dynamic(cpu_dev); + + platform_device_unregister(cpufreq_dt); +} + +/* + * Since the driver depends on clk-raspberrypi, which may return EPROBE_DEFER, + * all the activity is performed in the probe, which may be defered as well. + */ +static struct platform_driver raspberrypi_cpufreq_driver = { + .driver = { + .name = "raspberrypi-cpufreq", + }, + .probe = raspberrypi_cpufreq_probe, + .remove = raspberrypi_cpufreq_remove, +}; +module_platform_driver(raspberrypi_cpufreq_driver); + +MODULE_AUTHOR("Nicolas Saenz Julienne <nsaenzjulienne@suse.de"); +MODULE_DESCRIPTION("Raspberry Pi cpufreq driver"); +MODULE_LICENSE("GPL"); +MODULE_ALIAS("platform:raspberrypi-cpufreq"); diff --git a/drivers/cpufreq/rcpufreq_dt.rs b/drivers/cpufreq/rcpufreq_dt.rs new file mode 100644 index 000000000000..31e07f0279db --- /dev/null +++ b/drivers/cpufreq/rcpufreq_dt.rs @@ -0,0 +1,222 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Rust based implementation of the cpufreq-dt driver. + +use kernel::{ + c_str, + clk::Clk, + cpu, cpufreq, + cpumask::CpumaskVar, + device::{Core, Device}, + error::code::*, + macros::vtable, + module_platform_driver, of, opp, platform, + prelude::*, + str::CString, + sync::Arc, +}; + +/// Finds exact supply name from the OF node. +fn find_supply_name_exact(dev: &Device, name: &str) -> Option<CString> { + let prop_name = CString::try_from_fmt(fmt!("{name}-supply")).ok()?; + dev.fwnode()? + .property_present(&prop_name) + .then(|| CString::try_from_fmt(fmt!("{name}")).ok()) + .flatten() +} + +/// Finds supply name for the CPU from DT. +fn find_supply_names(dev: &Device, cpu: cpu::CpuId) -> Option<KVec<CString>> { + // Try "cpu0" for older DTs, fallback to "cpu". + (cpu.as_u32() == 0) + .then(|| find_supply_name_exact(dev, "cpu0")) + .flatten() + .or_else(|| find_supply_name_exact(dev, "cpu")) + .and_then(|name| kernel::kvec![name].ok()) +} + +/// Represents the cpufreq dt device. +struct CPUFreqDTDevice { + opp_table: opp::Table, + freq_table: opp::FreqTable, + _mask: CpumaskVar, + _token: Option<opp::ConfigToken>, + _clk: Clk, +} + +#[derive(Default)] +struct CPUFreqDTDriver; + +#[vtable] +impl opp::ConfigOps for CPUFreqDTDriver {} + +#[vtable] +impl cpufreq::Driver for CPUFreqDTDriver { + const NAME: &'static CStr = c_str!("cpufreq-dt"); + const FLAGS: u16 = cpufreq::flags::NEED_INITIAL_FREQ_CHECK | cpufreq::flags::IS_COOLING_DEV; + const BOOST_ENABLED: bool = true; + + type PData = Arc<CPUFreqDTDevice>; + + fn init(policy: &mut cpufreq::Policy) -> Result<Self::PData> { + let cpu = policy.cpu(); + // SAFETY: The CPU device is only used during init; it won't get hot-unplugged. The cpufreq + // core registers with CPU notifiers and the cpufreq core/driver won't use the CPU device, + // once the CPU is hot-unplugged. + let dev = unsafe { cpu::from_cpu(cpu)? }; + let mut mask = CpumaskVar::new_zero(GFP_KERNEL)?; + + mask.set(cpu); + + let token = find_supply_names(dev, cpu) + .map(|names| { + opp::Config::<Self>::new() + .set_regulator_names(names)? + .set(dev) + }) + .transpose()?; + + // Get OPP-sharing information from "operating-points-v2" bindings. + let fallback = match opp::Table::of_sharing_cpus(dev, &mut mask) { + Ok(()) => false, + Err(e) if e == ENOENT => { + // "operating-points-v2" not supported. If the platform hasn't + // set sharing CPUs, fallback to all CPUs share the `Policy` + // for backward compatibility. + opp::Table::sharing_cpus(dev, &mut mask).is_err() + } + Err(e) => return Err(e), + }; + + // Initialize OPP tables for all policy cpus. + // + // For platforms not using "operating-points-v2" bindings, we do this + // before updating policy cpus. Otherwise, we will end up creating + // duplicate OPPs for the CPUs. + // + // OPPs might be populated at runtime, don't fail for error here unless + // it is -EPROBE_DEFER. + let mut opp_table = match opp::Table::from_of_cpumask(dev, &mut mask) { + Ok(table) => table, + Err(e) => { + if e == EPROBE_DEFER { + return Err(e); + } + + // The table is added dynamically ? + opp::Table::from_dev(dev)? + } + }; + + // The OPP table must be initialized, statically or dynamically, by this point. + opp_table.opp_count()?; + + // Set sharing cpus for fallback scenario. + if fallback { + mask.setall(); + opp_table.set_sharing_cpus(&mut mask)?; + } + + let mut transition_latency = opp_table.max_transition_latency_ns() as u32; + if transition_latency == 0 { + transition_latency = cpufreq::DEFAULT_TRANSITION_LATENCY_NS; + } + + policy + .set_dvfs_possible_from_any_cpu(true) + .set_suspend_freq(opp_table.suspend_freq()) + .set_transition_latency_ns(transition_latency); + + let freq_table = opp_table.cpufreq_table()?; + // SAFETY: The `freq_table` is not dropped while it is getting used by the C code. + unsafe { policy.set_freq_table(&freq_table) }; + + // SAFETY: The returned `clk` is not dropped while it is getting used by the C code. + let clk = unsafe { policy.set_clk(dev, None)? }; + + mask.copy(policy.cpus()); + + Ok(Arc::new( + CPUFreqDTDevice { + opp_table, + freq_table, + _mask: mask, + _token: token, + _clk: clk, + }, + GFP_KERNEL, + )?) + } + + fn exit(_policy: &mut cpufreq::Policy, _data: Option<Self::PData>) -> Result { + Ok(()) + } + + fn online(_policy: &mut cpufreq::Policy) -> Result { + // We did light-weight tear down earlier, nothing to do here. + Ok(()) + } + + fn offline(_policy: &mut cpufreq::Policy) -> Result { + // Preserve policy->data and don't free resources on light-weight + // tear down. + Ok(()) + } + + fn suspend(policy: &mut cpufreq::Policy) -> Result { + policy.generic_suspend() + } + + fn verify(data: &mut cpufreq::PolicyData) -> Result { + data.generic_verify() + } + + fn target_index(policy: &mut cpufreq::Policy, index: cpufreq::TableIndex) -> Result { + let Some(data) = policy.data::<Self::PData>() else { + return Err(ENOENT); + }; + + let freq = data.freq_table.freq(index)?; + data.opp_table.set_rate(freq) + } + + fn get(policy: &mut cpufreq::Policy) -> Result<u32> { + policy.generic_get() + } + + fn set_boost(_policy: &mut cpufreq::Policy, _state: i32) -> Result { + Ok(()) + } + + fn register_em(policy: &mut cpufreq::Policy) { + policy.register_em_opp() + } +} + +kernel::of_device_table!( + OF_TABLE, + MODULE_OF_TABLE, + <CPUFreqDTDriver as platform::Driver>::IdInfo, + [(of::DeviceId::new(c_str!("operating-points-v2")), ())] +); + +impl platform::Driver for CPUFreqDTDriver { + type IdInfo = (); + const OF_ID_TABLE: Option<of::IdTable<Self::IdInfo>> = Some(&OF_TABLE); + + fn probe( + pdev: &platform::Device<Core>, + _id_info: Option<&Self::IdInfo>, + ) -> impl PinInit<Self, Error> { + cpufreq::Registration::<CPUFreqDTDriver>::new_foreign_owned(pdev.as_ref())?; + Ok(Self {}) + } +} + +module_platform_driver! { + type: CPUFreqDTDriver, + name: "cpufreq-dt", + authors: ["Viresh Kumar <viresh.kumar@linaro.org>"], + description: "Generic CPUFreq DT driver", + license: "GPL v2", +} diff --git a/drivers/cpufreq/s3c2410-cpufreq.c b/drivers/cpufreq/s3c2410-cpufreq.c deleted file mode 100644 index cfa0dd8723ec..000000000000 --- a/drivers/cpufreq/s3c2410-cpufreq.c +++ /dev/null @@ -1,160 +0,0 @@ -/* - * Copyright (c) 2006-2008 Simtec Electronics - * http://armlinux.simtec.co.uk/ - * Ben Dooks <ben@simtec.co.uk> - * - * S3C2410 CPU Frequency scaling - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. -*/ - -#include <linux/init.h> -#include <linux/module.h> -#include <linux/interrupt.h> -#include <linux/ioport.h> -#include <linux/cpufreq.h> -#include <linux/device.h> -#include <linux/clk.h> -#include <linux/err.h> -#include <linux/io.h> - -#include <asm/mach/arch.h> -#include <asm/mach/map.h> - -#include <mach/regs-clock.h> - -#include <plat/cpu.h> -#include <plat/clock.h> -#include <plat/cpu-freq-core.h> - -/* Note, 2410A has an extra mode for 1:4:4 ratio, bit 2 of CLKDIV */ - -static void s3c2410_cpufreq_setdivs(struct s3c_cpufreq_config *cfg) -{ - u32 clkdiv = 0; - - if (cfg->divs.h_divisor == 2) - clkdiv |= S3C2410_CLKDIVN_HDIVN; - - if (cfg->divs.p_divisor != cfg->divs.h_divisor) - clkdiv |= S3C2410_CLKDIVN_PDIVN; - - __raw_writel(clkdiv, S3C2410_CLKDIVN); -} - -static int s3c2410_cpufreq_calcdivs(struct s3c_cpufreq_config *cfg) -{ - unsigned long hclk, fclk, pclk; - unsigned int hdiv, pdiv; - unsigned long hclk_max; - - fclk = cfg->freq.fclk; - hclk_max = cfg->max.hclk; - - cfg->freq.armclk = fclk; - - s3c_freq_dbg("%s: fclk is %lu, max hclk %lu\n", - __func__, fclk, hclk_max); - - hdiv = (fclk > cfg->max.hclk) ? 2 : 1; - hclk = fclk / hdiv; - - if (hclk > cfg->max.hclk) { - s3c_freq_dbg("%s: hclk too big\n", __func__); - return -EINVAL; - } - - pdiv = (hclk > cfg->max.pclk) ? 2 : 1; - pclk = hclk / pdiv; - - if (pclk > cfg->max.pclk) { - s3c_freq_dbg("%s: pclk too big\n", __func__); - return -EINVAL; - } - - pdiv *= hdiv; - - /* record the result */ - cfg->divs.p_divisor = pdiv; - cfg->divs.h_divisor = hdiv; - - return 0; -} - -static struct s3c_cpufreq_info s3c2410_cpufreq_info = { - .max = { - .fclk = 200000000, - .hclk = 100000000, - .pclk = 50000000, - }, - - /* transition latency is about 5ms worst-case, so - * set 10ms to be sure */ - .latency = 10000000, - - .locktime_m = 150, - .locktime_u = 150, - .locktime_bits = 12, - - .need_pll = 1, - - .name = "s3c2410", - .calc_iotiming = s3c2410_iotiming_calc, - .set_iotiming = s3c2410_iotiming_set, - .get_iotiming = s3c2410_iotiming_get, - .resume_clocks = s3c2410_setup_clocks, - - .set_fvco = s3c2410_set_fvco, - .set_refresh = s3c2410_cpufreq_setrefresh, - .set_divs = s3c2410_cpufreq_setdivs, - .calc_divs = s3c2410_cpufreq_calcdivs, - - .debug_io_show = s3c_cpufreq_debugfs_call(s3c2410_iotiming_debugfs), -}; - -static int s3c2410_cpufreq_add(struct device *dev, - struct subsys_interface *sif) -{ - return s3c_cpufreq_register(&s3c2410_cpufreq_info); -} - -static struct subsys_interface s3c2410_cpufreq_interface = { - .name = "s3c2410_cpufreq", - .subsys = &s3c2410_subsys, - .add_dev = s3c2410_cpufreq_add, -}; - -static int __init s3c2410_cpufreq_init(void) -{ - return subsys_interface_register(&s3c2410_cpufreq_interface); -} -arch_initcall(s3c2410_cpufreq_init); - -static int s3c2410a_cpufreq_add(struct device *dev, - struct subsys_interface *sif) -{ - /* alter the maximum freq settings for S3C2410A. If a board knows - * it only has a maximum of 200, then it should register its own - * limits. */ - - s3c2410_cpufreq_info.max.fclk = 266000000; - s3c2410_cpufreq_info.max.hclk = 133000000; - s3c2410_cpufreq_info.max.pclk = 66500000; - s3c2410_cpufreq_info.name = "s3c2410a"; - - return s3c2410_cpufreq_add(dev, sif); -} - -static struct subsys_interface s3c2410a_cpufreq_interface = { - .name = "s3c2410a_cpufreq", - .subsys = &s3c2410a_subsys, - .add_dev = s3c2410a_cpufreq_add, -}; - -static int __init s3c2410a_cpufreq_init(void) -{ - return subsys_interface_register(&s3c2410a_cpufreq_interface); -} -arch_initcall(s3c2410a_cpufreq_init); diff --git a/drivers/cpufreq/s3c2412-cpufreq.c b/drivers/cpufreq/s3c2412-cpufreq.c deleted file mode 100644 index 4645b4898996..000000000000 --- a/drivers/cpufreq/s3c2412-cpufreq.c +++ /dev/null @@ -1,257 +0,0 @@ -/* - * Copyright 2008 Simtec Electronics - * http://armlinux.simtec.co.uk/ - * Ben Dooks <ben@simtec.co.uk> - * - * S3C2412 CPU Frequency scalling - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. -*/ - -#include <linux/init.h> -#include <linux/module.h> -#include <linux/interrupt.h> -#include <linux/ioport.h> -#include <linux/cpufreq.h> -#include <linux/device.h> -#include <linux/delay.h> -#include <linux/clk.h> -#include <linux/err.h> -#include <linux/io.h> - -#include <asm/mach/arch.h> -#include <asm/mach/map.h> - -#include <mach/regs-clock.h> -#include <mach/s3c2412.h> - -#include <plat/cpu.h> -#include <plat/clock.h> -#include <plat/cpu-freq-core.h> - -/* our clock resources. */ -static struct clk *xtal; -static struct clk *fclk; -static struct clk *hclk; -static struct clk *armclk; - -/* HDIV: 1, 2, 3, 4, 6, 8 */ - -static int s3c2412_cpufreq_calcdivs(struct s3c_cpufreq_config *cfg) -{ - unsigned int hdiv, pdiv, armdiv, dvs; - unsigned long hclk, fclk, armclk, armdiv_clk; - unsigned long hclk_max; - - fclk = cfg->freq.fclk; - armclk = cfg->freq.armclk; - hclk_max = cfg->max.hclk; - - /* We can't run hclk above armclk as at the best we have to - * have armclk and hclk in dvs mode. */ - - if (hclk_max > armclk) - hclk_max = armclk; - - s3c_freq_dbg("%s: fclk=%lu, armclk=%lu, hclk_max=%lu\n", - __func__, fclk, armclk, hclk_max); - s3c_freq_dbg("%s: want f=%lu, arm=%lu, h=%lu, p=%lu\n", - __func__, cfg->freq.fclk, cfg->freq.armclk, - cfg->freq.hclk, cfg->freq.pclk); - - armdiv = fclk / armclk; - - if (armdiv < 1) - armdiv = 1; - if (armdiv > 2) - armdiv = 2; - - cfg->divs.arm_divisor = armdiv; - armdiv_clk = fclk / armdiv; - - hdiv = armdiv_clk / hclk_max; - if (hdiv < 1) - hdiv = 1; - - cfg->freq.hclk = hclk = armdiv_clk / hdiv; - - /* set dvs depending on whether we reached armclk or not. */ - cfg->divs.dvs = dvs = armclk < armdiv_clk; - - /* update the actual armclk we achieved. */ - cfg->freq.armclk = dvs ? hclk : armdiv_clk; - - s3c_freq_dbg("%s: armclk %lu, hclk %lu, armdiv %d, hdiv %d, dvs %d\n", - __func__, armclk, hclk, armdiv, hdiv, cfg->divs.dvs); - - if (hdiv > 4) - goto invalid; - - pdiv = (hclk > cfg->max.pclk) ? 2 : 1; - - if ((hclk / pdiv) > cfg->max.pclk) - pdiv++; - - cfg->freq.pclk = hclk / pdiv; - - s3c_freq_dbg("%s: pdiv %d\n", __func__, pdiv); - - if (pdiv > 2) - goto invalid; - - pdiv *= hdiv; - - /* store the result, and then return */ - - cfg->divs.h_divisor = hdiv * armdiv; - cfg->divs.p_divisor = pdiv * armdiv; - - return 0; - -invalid: - return -EINVAL; -} - -static void s3c2412_cpufreq_setdivs(struct s3c_cpufreq_config *cfg) -{ - unsigned long clkdiv; - unsigned long olddiv; - - olddiv = clkdiv = __raw_readl(S3C2410_CLKDIVN); - - /* clear off current clock info */ - - clkdiv &= ~S3C2412_CLKDIVN_ARMDIVN; - clkdiv &= ~S3C2412_CLKDIVN_HDIVN_MASK; - clkdiv &= ~S3C2412_CLKDIVN_PDIVN; - - if (cfg->divs.arm_divisor == 2) - clkdiv |= S3C2412_CLKDIVN_ARMDIVN; - - clkdiv |= ((cfg->divs.h_divisor / cfg->divs.arm_divisor) - 1); - - if (cfg->divs.p_divisor != cfg->divs.h_divisor) - clkdiv |= S3C2412_CLKDIVN_PDIVN; - - s3c_freq_dbg("%s: div %08lx => %08lx\n", __func__, olddiv, clkdiv); - __raw_writel(clkdiv, S3C2410_CLKDIVN); - - clk_set_parent(armclk, cfg->divs.dvs ? hclk : fclk); -} - -static void s3c2412_cpufreq_setrefresh(struct s3c_cpufreq_config *cfg) -{ - struct s3c_cpufreq_board *board = cfg->board; - unsigned long refresh; - - s3c_freq_dbg("%s: refresh %u ns, hclk %lu\n", __func__, - board->refresh, cfg->freq.hclk); - - /* Reduce both the refresh time (in ns) and the frequency (in MHz) - * by 10 each to ensure that we do not overflow 32 bit numbers. This - * should work for HCLK up to 133MHz and refresh period up to 30usec. - */ - - refresh = (board->refresh / 10); - refresh *= (cfg->freq.hclk / 100); - refresh /= (1 * 1000 * 1000); /* 10^6 */ - - s3c_freq_dbg("%s: setting refresh 0x%08lx\n", __func__, refresh); - __raw_writel(refresh, S3C2412_REFRESH); -} - -/* set the default cpu frequency information, based on an 200MHz part - * as we have no other way of detecting the speed rating in software. - */ - -static struct s3c_cpufreq_info s3c2412_cpufreq_info = { - .max = { - .fclk = 200000000, - .hclk = 100000000, - .pclk = 50000000, - }, - - .latency = 5000000, /* 5ms */ - - .locktime_m = 150, - .locktime_u = 150, - .locktime_bits = 16, - - .name = "s3c2412", - .set_refresh = s3c2412_cpufreq_setrefresh, - .set_divs = s3c2412_cpufreq_setdivs, - .calc_divs = s3c2412_cpufreq_calcdivs, - - .calc_iotiming = s3c2412_iotiming_calc, - .set_iotiming = s3c2412_iotiming_set, - .get_iotiming = s3c2412_iotiming_get, - - .resume_clocks = s3c2412_setup_clocks, - - .debug_io_show = s3c_cpufreq_debugfs_call(s3c2412_iotiming_debugfs), -}; - -static int s3c2412_cpufreq_add(struct device *dev, - struct subsys_interface *sif) -{ - unsigned long fclk_rate; - - hclk = clk_get(NULL, "hclk"); - if (IS_ERR(hclk)) { - printk(KERN_ERR "%s: cannot find hclk clock\n", __func__); - return -ENOENT; - } - - fclk = clk_get(NULL, "fclk"); - if (IS_ERR(fclk)) { - printk(KERN_ERR "%s: cannot find fclk clock\n", __func__); - goto err_fclk; - } - - fclk_rate = clk_get_rate(fclk); - if (fclk_rate > 200000000) { - printk(KERN_INFO - "%s: fclk %ld MHz, assuming 266MHz capable part\n", - __func__, fclk_rate / 1000000); - s3c2412_cpufreq_info.max.fclk = 266000000; - s3c2412_cpufreq_info.max.hclk = 133000000; - s3c2412_cpufreq_info.max.pclk = 66000000; - } - - armclk = clk_get(NULL, "armclk"); - if (IS_ERR(armclk)) { - printk(KERN_ERR "%s: cannot find arm clock\n", __func__); - goto err_armclk; - } - - xtal = clk_get(NULL, "xtal"); - if (IS_ERR(xtal)) { - printk(KERN_ERR "%s: cannot find xtal clock\n", __func__); - goto err_xtal; - } - - return s3c_cpufreq_register(&s3c2412_cpufreq_info); - -err_xtal: - clk_put(armclk); -err_armclk: - clk_put(fclk); -err_fclk: - clk_put(hclk); - - return -ENOENT; -} - -static struct subsys_interface s3c2412_cpufreq_interface = { - .name = "s3c2412_cpufreq", - .subsys = &s3c2412_subsys, - .add_dev = s3c2412_cpufreq_add, -}; - -static int s3c2412_cpufreq_init(void) -{ - return subsys_interface_register(&s3c2412_cpufreq_interface); -} -arch_initcall(s3c2412_cpufreq_init); diff --git a/drivers/cpufreq/s3c2416-cpufreq.c b/drivers/cpufreq/s3c2416-cpufreq.c deleted file mode 100644 index ce5b9fca9c18..000000000000 --- a/drivers/cpufreq/s3c2416-cpufreq.c +++ /dev/null @@ -1,541 +0,0 @@ -/* - * S3C2416/2450 CPUfreq Support - * - * Copyright 2011 Heiko Stuebner <heiko@sntech.de> - * - * based on s3c64xx_cpufreq.c - * - * Copyright 2009 Wolfson Microelectronics plc - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - */ - -#include <linux/kernel.h> -#include <linux/types.h> -#include <linux/init.h> -#include <linux/cpufreq.h> -#include <linux/clk.h> -#include <linux/err.h> -#include <linux/regulator/consumer.h> -#include <linux/reboot.h> -#include <linux/module.h> - -static DEFINE_MUTEX(cpufreq_lock); - -struct s3c2416_data { - struct clk *armdiv; - struct clk *armclk; - struct clk *hclk; - - unsigned long regulator_latency; -#ifdef CONFIG_ARM_S3C2416_CPUFREQ_VCORESCALE - struct regulator *vddarm; -#endif - - struct cpufreq_frequency_table *freq_table; - - bool is_dvs; - bool disable_dvs; -}; - -static struct s3c2416_data s3c2416_cpufreq; - -struct s3c2416_dvfs { - unsigned int vddarm_min; - unsigned int vddarm_max; -}; - -/* pseudo-frequency for dvs mode */ -#define FREQ_DVS 132333 - -/* frequency to sleep and reboot in - * it's essential to leave dvs, as some boards do not reconfigure the - * regulator on reboot - */ -#define FREQ_SLEEP 133333 - -/* Sources for the ARMCLK */ -#define SOURCE_HCLK 0 -#define SOURCE_ARMDIV 1 - -#ifdef CONFIG_ARM_S3C2416_CPUFREQ_VCORESCALE -/* S3C2416 only supports changing the voltage in the dvs-mode. - * Voltages down to 1.0V seem to work, so we take what the regulator - * can get us. - */ -static struct s3c2416_dvfs s3c2416_dvfs_table[] = { - [SOURCE_HCLK] = { 950000, 1250000 }, - [SOURCE_ARMDIV] = { 1250000, 1350000 }, -}; -#endif - -static struct cpufreq_frequency_table s3c2416_freq_table[] = { - { SOURCE_HCLK, FREQ_DVS }, - { SOURCE_ARMDIV, 133333 }, - { SOURCE_ARMDIV, 266666 }, - { SOURCE_ARMDIV, 400000 }, - { 0, CPUFREQ_TABLE_END }, -}; - -static struct cpufreq_frequency_table s3c2450_freq_table[] = { - { SOURCE_HCLK, FREQ_DVS }, - { SOURCE_ARMDIV, 133500 }, - { SOURCE_ARMDIV, 267000 }, - { SOURCE_ARMDIV, 534000 }, - { 0, CPUFREQ_TABLE_END }, -}; - -static int s3c2416_cpufreq_verify_speed(struct cpufreq_policy *policy) -{ - struct s3c2416_data *s3c_freq = &s3c2416_cpufreq; - - if (policy->cpu != 0) - return -EINVAL; - - return cpufreq_frequency_table_verify(policy, s3c_freq->freq_table); -} - -static unsigned int s3c2416_cpufreq_get_speed(unsigned int cpu) -{ - struct s3c2416_data *s3c_freq = &s3c2416_cpufreq; - - if (cpu != 0) - return 0; - - /* return our pseudo-frequency when in dvs mode */ - if (s3c_freq->is_dvs) - return FREQ_DVS; - - return clk_get_rate(s3c_freq->armclk) / 1000; -} - -static int s3c2416_cpufreq_set_armdiv(struct s3c2416_data *s3c_freq, - unsigned int freq) -{ - int ret; - - if (clk_get_rate(s3c_freq->armdiv) / 1000 != freq) { - ret = clk_set_rate(s3c_freq->armdiv, freq * 1000); - if (ret < 0) { - pr_err("cpufreq: Failed to set armdiv rate %dkHz: %d\n", - freq, ret); - return ret; - } - } - - return 0; -} - -static int s3c2416_cpufreq_enter_dvs(struct s3c2416_data *s3c_freq, int idx) -{ -#ifdef CONFIG_ARM_S3C2416_CPUFREQ_VCORESCALE - struct s3c2416_dvfs *dvfs; -#endif - int ret; - - if (s3c_freq->is_dvs) { - pr_debug("cpufreq: already in dvs mode, nothing to do\n"); - return 0; - } - - pr_debug("cpufreq: switching armclk to hclk (%lukHz)\n", - clk_get_rate(s3c_freq->hclk) / 1000); - ret = clk_set_parent(s3c_freq->armclk, s3c_freq->hclk); - if (ret < 0) { - pr_err("cpufreq: Failed to switch armclk to hclk: %d\n", ret); - return ret; - } - -#ifdef CONFIG_ARM_S3C2416_CPUFREQ_VCORESCALE - /* changing the core voltage is only allowed when in dvs mode */ - if (s3c_freq->vddarm) { - dvfs = &s3c2416_dvfs_table[idx]; - - pr_debug("cpufreq: setting regulator to %d-%d\n", - dvfs->vddarm_min, dvfs->vddarm_max); - ret = regulator_set_voltage(s3c_freq->vddarm, - dvfs->vddarm_min, - dvfs->vddarm_max); - - /* when lowering the voltage failed, there is nothing to do */ - if (ret != 0) - pr_err("cpufreq: Failed to set VDDARM: %d\n", ret); - } -#endif - - s3c_freq->is_dvs = 1; - - return 0; -} - -static int s3c2416_cpufreq_leave_dvs(struct s3c2416_data *s3c_freq, int idx) -{ -#ifdef CONFIG_ARM_S3C2416_CPUFREQ_VCORESCALE - struct s3c2416_dvfs *dvfs; -#endif - int ret; - - if (!s3c_freq->is_dvs) { - pr_debug("cpufreq: not in dvs mode, so can't leave\n"); - return 0; - } - -#ifdef CONFIG_ARM_S3C2416_CPUFREQ_VCORESCALE - if (s3c_freq->vddarm) { - dvfs = &s3c2416_dvfs_table[idx]; - - pr_debug("cpufreq: setting regulator to %d-%d\n", - dvfs->vddarm_min, dvfs->vddarm_max); - ret = regulator_set_voltage(s3c_freq->vddarm, - dvfs->vddarm_min, - dvfs->vddarm_max); - if (ret != 0) { - pr_err("cpufreq: Failed to set VDDARM: %d\n", ret); - return ret; - } - } -#endif - - /* force armdiv to hclk frequency for transition from dvs*/ - if (clk_get_rate(s3c_freq->armdiv) > clk_get_rate(s3c_freq->hclk)) { - pr_debug("cpufreq: force armdiv to hclk frequency (%lukHz)\n", - clk_get_rate(s3c_freq->hclk) / 1000); - ret = s3c2416_cpufreq_set_armdiv(s3c_freq, - clk_get_rate(s3c_freq->hclk) / 1000); - if (ret < 0) { - pr_err("cpufreq: Failed to set the armdiv to %lukHz: %d\n", - clk_get_rate(s3c_freq->hclk) / 1000, ret); - return ret; - } - } - - pr_debug("cpufreq: switching armclk parent to armdiv (%lukHz)\n", - clk_get_rate(s3c_freq->armdiv) / 1000); - - ret = clk_set_parent(s3c_freq->armclk, s3c_freq->armdiv); - if (ret < 0) { - pr_err("cpufreq: Failed to switch armclk clock parent to armdiv: %d\n", - ret); - return ret; - } - - s3c_freq->is_dvs = 0; - - return 0; -} - -static int s3c2416_cpufreq_set_target(struct cpufreq_policy *policy, - unsigned int target_freq, - unsigned int relation) -{ - struct s3c2416_data *s3c_freq = &s3c2416_cpufreq; - struct cpufreq_freqs freqs; - int idx, ret, to_dvs = 0; - unsigned int i; - - mutex_lock(&cpufreq_lock); - - pr_debug("cpufreq: to %dKHz, relation %d\n", target_freq, relation); - - ret = cpufreq_frequency_table_target(policy, s3c_freq->freq_table, - target_freq, relation, &i); - if (ret != 0) - goto out; - - idx = s3c_freq->freq_table[i].driver_data; - - if (idx == SOURCE_HCLK) - to_dvs = 1; - - /* switching to dvs when it's not allowed */ - if (to_dvs && s3c_freq->disable_dvs) { - pr_debug("cpufreq: entering dvs mode not allowed\n"); - ret = -EINVAL; - goto out; - } - - freqs.flags = 0; - freqs.old = s3c_freq->is_dvs ? FREQ_DVS - : clk_get_rate(s3c_freq->armclk) / 1000; - - /* When leavin dvs mode, always switch the armdiv to the hclk rate - * The S3C2416 has stability issues when switching directly to - * higher frequencies. - */ - freqs.new = (s3c_freq->is_dvs && !to_dvs) - ? clk_get_rate(s3c_freq->hclk) / 1000 - : s3c_freq->freq_table[i].frequency; - - pr_debug("cpufreq: Transition %d-%dkHz\n", freqs.old, freqs.new); - - if (!to_dvs && freqs.old == freqs.new) - goto out; - - cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE); - - if (to_dvs) { - pr_debug("cpufreq: enter dvs\n"); - ret = s3c2416_cpufreq_enter_dvs(s3c_freq, idx); - } else if (s3c_freq->is_dvs) { - pr_debug("cpufreq: leave dvs\n"); - ret = s3c2416_cpufreq_leave_dvs(s3c_freq, idx); - } else { - pr_debug("cpufreq: change armdiv to %dkHz\n", freqs.new); - ret = s3c2416_cpufreq_set_armdiv(s3c_freq, freqs.new); - } - - cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); - -out: - mutex_unlock(&cpufreq_lock); - - return ret; -} - -#ifdef CONFIG_ARM_S3C2416_CPUFREQ_VCORESCALE -static void __init s3c2416_cpufreq_cfg_regulator(struct s3c2416_data *s3c_freq) -{ - int count, v, i, found; - struct cpufreq_frequency_table *freq; - struct s3c2416_dvfs *dvfs; - - count = regulator_count_voltages(s3c_freq->vddarm); - if (count < 0) { - pr_err("cpufreq: Unable to check supported voltages\n"); - return; - } - - freq = s3c_freq->freq_table; - while (count > 0 && freq->frequency != CPUFREQ_TABLE_END) { - if (freq->frequency == CPUFREQ_ENTRY_INVALID) - continue; - - dvfs = &s3c2416_dvfs_table[freq->driver_data]; - found = 0; - - /* Check only the min-voltage, more is always ok on S3C2416 */ - for (i = 0; i < count; i++) { - v = regulator_list_voltage(s3c_freq->vddarm, i); - if (v >= dvfs->vddarm_min) - found = 1; - } - - if (!found) { - pr_debug("cpufreq: %dkHz unsupported by regulator\n", - freq->frequency); - freq->frequency = CPUFREQ_ENTRY_INVALID; - } - - freq++; - } - - /* Guessed */ - s3c_freq->regulator_latency = 1 * 1000 * 1000; -} -#endif - -static int s3c2416_cpufreq_reboot_notifier_evt(struct notifier_block *this, - unsigned long event, void *ptr) -{ - struct s3c2416_data *s3c_freq = &s3c2416_cpufreq; - int ret; - - mutex_lock(&cpufreq_lock); - - /* disable further changes */ - s3c_freq->disable_dvs = 1; - - mutex_unlock(&cpufreq_lock); - - /* some boards don't reconfigure the regulator on reboot, which - * could lead to undervolting the cpu when the clock is reset. - * Therefore we always leave the DVS mode on reboot. - */ - if (s3c_freq->is_dvs) { - pr_debug("cpufreq: leave dvs on reboot\n"); - ret = cpufreq_driver_target(cpufreq_cpu_get(0), FREQ_SLEEP, 0); - if (ret < 0) - return NOTIFY_BAD; - } - - return NOTIFY_DONE; -} - -static struct notifier_block s3c2416_cpufreq_reboot_notifier = { - .notifier_call = s3c2416_cpufreq_reboot_notifier_evt, -}; - -static int __init s3c2416_cpufreq_driver_init(struct cpufreq_policy *policy) -{ - struct s3c2416_data *s3c_freq = &s3c2416_cpufreq; - struct cpufreq_frequency_table *freq; - struct clk *msysclk; - unsigned long rate; - int ret; - - if (policy->cpu != 0) - return -EINVAL; - - msysclk = clk_get(NULL, "msysclk"); - if (IS_ERR(msysclk)) { - ret = PTR_ERR(msysclk); - pr_err("cpufreq: Unable to obtain msysclk: %d\n", ret); - return ret; - } - - /* - * S3C2416 and S3C2450 share the same processor-ID and also provide no - * other means to distinguish them other than through the rate of - * msysclk. On S3C2416 msysclk runs at 800MHz and on S3C2450 at 533MHz. - */ - rate = clk_get_rate(msysclk); - if (rate == 800 * 1000 * 1000) { - pr_info("cpufreq: msysclk running at %lukHz, using S3C2416 frequency table\n", - rate / 1000); - s3c_freq->freq_table = s3c2416_freq_table; - policy->cpuinfo.max_freq = 400000; - } else if (rate / 1000 == 534000) { - pr_info("cpufreq: msysclk running at %lukHz, using S3C2450 frequency table\n", - rate / 1000); - s3c_freq->freq_table = s3c2450_freq_table; - policy->cpuinfo.max_freq = 534000; - } - - /* not needed anymore */ - clk_put(msysclk); - - if (s3c_freq->freq_table == NULL) { - pr_err("cpufreq: No frequency information for this CPU, msysclk at %lukHz\n", - rate / 1000); - return -ENODEV; - } - - s3c_freq->is_dvs = 0; - - s3c_freq->armdiv = clk_get(NULL, "armdiv"); - if (IS_ERR(s3c_freq->armdiv)) { - ret = PTR_ERR(s3c_freq->armdiv); - pr_err("cpufreq: Unable to obtain ARMDIV: %d\n", ret); - return ret; - } - - s3c_freq->hclk = clk_get(NULL, "hclk"); - if (IS_ERR(s3c_freq->hclk)) { - ret = PTR_ERR(s3c_freq->hclk); - pr_err("cpufreq: Unable to obtain HCLK: %d\n", ret); - goto err_hclk; - } - - /* chech hclk rate, we only support the common 133MHz for now - * hclk could also run at 66MHz, but this not often used - */ - rate = clk_get_rate(s3c_freq->hclk); - if (rate < 133 * 1000 * 1000) { - pr_err("cpufreq: HCLK not at 133MHz\n"); - clk_put(s3c_freq->hclk); - ret = -EINVAL; - goto err_armclk; - } - - s3c_freq->armclk = clk_get(NULL, "armclk"); - if (IS_ERR(s3c_freq->armclk)) { - ret = PTR_ERR(s3c_freq->armclk); - pr_err("cpufreq: Unable to obtain ARMCLK: %d\n", ret); - goto err_armclk; - } - -#ifdef CONFIG_ARM_S3C2416_CPUFREQ_VCORESCALE - s3c_freq->vddarm = regulator_get(NULL, "vddarm"); - if (IS_ERR(s3c_freq->vddarm)) { - ret = PTR_ERR(s3c_freq->vddarm); - pr_err("cpufreq: Failed to obtain VDDARM: %d\n", ret); - goto err_vddarm; - } - - s3c2416_cpufreq_cfg_regulator(s3c_freq); -#else - s3c_freq->regulator_latency = 0; -#endif - - freq = s3c_freq->freq_table; - while (freq->frequency != CPUFREQ_TABLE_END) { - /* special handling for dvs mode */ - if (freq->driver_data == 0) { - if (!s3c_freq->hclk) { - pr_debug("cpufreq: %dkHz unsupported as it would need unavailable dvs mode\n", - freq->frequency); - freq->frequency = CPUFREQ_ENTRY_INVALID; - } else { - freq++; - continue; - } - } - - /* Check for frequencies we can generate */ - rate = clk_round_rate(s3c_freq->armdiv, - freq->frequency * 1000); - rate /= 1000; - if (rate != freq->frequency) { - pr_debug("cpufreq: %dkHz unsupported by clock (clk_round_rate return %lu)\n", - freq->frequency, rate); - freq->frequency = CPUFREQ_ENTRY_INVALID; - } - - freq++; - } - - policy->cur = clk_get_rate(s3c_freq->armclk) / 1000; - - /* Datasheet says PLL stabalisation time must be at least 300us, - * so but add some fudge. (reference in LOCKCON0 register description) - */ - policy->cpuinfo.transition_latency = (500 * 1000) + - s3c_freq->regulator_latency; - - ret = cpufreq_frequency_table_cpuinfo(policy, s3c_freq->freq_table); - if (ret) - goto err_freq_table; - - cpufreq_frequency_table_get_attr(s3c_freq->freq_table, 0); - - register_reboot_notifier(&s3c2416_cpufreq_reboot_notifier); - - return 0; - -err_freq_table: -#ifdef CONFIG_ARM_S3C2416_CPUFREQ_VCORESCALE - regulator_put(s3c_freq->vddarm); -err_vddarm: -#endif - clk_put(s3c_freq->armclk); -err_armclk: - clk_put(s3c_freq->hclk); -err_hclk: - clk_put(s3c_freq->armdiv); - - return ret; -} - -static struct freq_attr *s3c2416_cpufreq_attr[] = { - &cpufreq_freq_attr_scaling_available_freqs, - NULL, -}; - -static struct cpufreq_driver s3c2416_cpufreq_driver = { - .owner = THIS_MODULE, - .flags = 0, - .verify = s3c2416_cpufreq_verify_speed, - .target = s3c2416_cpufreq_set_target, - .get = s3c2416_cpufreq_get_speed, - .init = s3c2416_cpufreq_driver_init, - .name = "s3c2416", - .attr = s3c2416_cpufreq_attr, -}; - -static int __init s3c2416_cpufreq_init(void) -{ - return cpufreq_register_driver(&s3c2416_cpufreq_driver); -} -module_init(s3c2416_cpufreq_init); diff --git a/drivers/cpufreq/s3c2440-cpufreq.c b/drivers/cpufreq/s3c2440-cpufreq.c deleted file mode 100644 index 72b2cc8a5a85..000000000000 --- a/drivers/cpufreq/s3c2440-cpufreq.c +++ /dev/null @@ -1,312 +0,0 @@ -/* - * Copyright (c) 2006-2009 Simtec Electronics - * http://armlinux.simtec.co.uk/ - * Ben Dooks <ben@simtec.co.uk> - * Vincent Sanders <vince@simtec.co.uk> - * - * S3C2440/S3C2442 CPU Frequency scaling - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. -*/ - -#include <linux/init.h> -#include <linux/module.h> -#include <linux/interrupt.h> -#include <linux/ioport.h> -#include <linux/cpufreq.h> -#include <linux/device.h> -#include <linux/delay.h> -#include <linux/clk.h> -#include <linux/err.h> -#include <linux/io.h> - -#include <mach/hardware.h> - -#include <asm/mach/arch.h> -#include <asm/mach/map.h> - -#include <mach/regs-clock.h> - -#include <plat/cpu.h> -#include <plat/cpu-freq-core.h> -#include <plat/clock.h> - -static struct clk *xtal; -static struct clk *fclk; -static struct clk *hclk; -static struct clk *armclk; - -/* HDIV: 1, 2, 3, 4, 6, 8 */ - -static inline int within_khz(unsigned long a, unsigned long b) -{ - long diff = a - b; - - return (diff >= -1000 && diff <= 1000); -} - -/** - * s3c2440_cpufreq_calcdivs - calculate divider settings - * @cfg: The cpu frequency settings. - * - * Calcualte the divider values for the given frequency settings - * specified in @cfg. The values are stored in @cfg for later use - * by the relevant set routine if the request settings can be reached. - */ -int s3c2440_cpufreq_calcdivs(struct s3c_cpufreq_config *cfg) -{ - unsigned int hdiv, pdiv; - unsigned long hclk, fclk, armclk; - unsigned long hclk_max; - - fclk = cfg->freq.fclk; - armclk = cfg->freq.armclk; - hclk_max = cfg->max.hclk; - - s3c_freq_dbg("%s: fclk is %lu, armclk %lu, max hclk %lu\n", - __func__, fclk, armclk, hclk_max); - - if (armclk > fclk) { - printk(KERN_WARNING "%s: armclk > fclk\n", __func__); - armclk = fclk; - } - - /* if we are in DVS, we need HCLK to be <= ARMCLK */ - if (armclk < fclk && armclk < hclk_max) - hclk_max = armclk; - - for (hdiv = 1; hdiv < 9; hdiv++) { - if (hdiv == 5 || hdiv == 7) - hdiv++; - - hclk = (fclk / hdiv); - if (hclk <= hclk_max || within_khz(hclk, hclk_max)) - break; - } - - s3c_freq_dbg("%s: hclk %lu, div %d\n", __func__, hclk, hdiv); - - if (hdiv > 8) - goto invalid; - - pdiv = (hclk > cfg->max.pclk) ? 2 : 1; - - if ((hclk / pdiv) > cfg->max.pclk) - pdiv++; - - s3c_freq_dbg("%s: pdiv %d\n", __func__, pdiv); - - if (pdiv > 2) - goto invalid; - - pdiv *= hdiv; - - /* calculate a valid armclk */ - - if (armclk < hclk) - armclk = hclk; - - /* if we're running armclk lower than fclk, this really means - * that the system should go into dvs mode, which means that - * armclk is connected to hclk. */ - if (armclk < fclk) { - cfg->divs.dvs = 1; - armclk = hclk; - } else - cfg->divs.dvs = 0; - - cfg->freq.armclk = armclk; - - /* store the result, and then return */ - - cfg->divs.h_divisor = hdiv; - cfg->divs.p_divisor = pdiv; - - return 0; - - invalid: - return -EINVAL; -} - -#define CAMDIVN_HCLK_HALF (S3C2440_CAMDIVN_HCLK3_HALF | \ - S3C2440_CAMDIVN_HCLK4_HALF) - -/** - * s3c2440_cpufreq_setdivs - set the cpu frequency divider settings - * @cfg: The cpu frequency settings. - * - * Set the divisors from the settings in @cfg, which where generated - * during the calculation phase by s3c2440_cpufreq_calcdivs(). - */ -static void s3c2440_cpufreq_setdivs(struct s3c_cpufreq_config *cfg) -{ - unsigned long clkdiv, camdiv; - - s3c_freq_dbg("%s: divsiors: h=%d, p=%d\n", __func__, - cfg->divs.h_divisor, cfg->divs.p_divisor); - - clkdiv = __raw_readl(S3C2410_CLKDIVN); - camdiv = __raw_readl(S3C2440_CAMDIVN); - - clkdiv &= ~(S3C2440_CLKDIVN_HDIVN_MASK | S3C2440_CLKDIVN_PDIVN); - camdiv &= ~CAMDIVN_HCLK_HALF; - - switch (cfg->divs.h_divisor) { - case 1: - clkdiv |= S3C2440_CLKDIVN_HDIVN_1; - break; - - case 2: - clkdiv |= S3C2440_CLKDIVN_HDIVN_2; - break; - - case 6: - camdiv |= S3C2440_CAMDIVN_HCLK3_HALF; - case 3: - clkdiv |= S3C2440_CLKDIVN_HDIVN_3_6; - break; - - case 8: - camdiv |= S3C2440_CAMDIVN_HCLK4_HALF; - case 4: - clkdiv |= S3C2440_CLKDIVN_HDIVN_4_8; - break; - - default: - BUG(); /* we don't expect to get here. */ - } - - if (cfg->divs.p_divisor != cfg->divs.h_divisor) - clkdiv |= S3C2440_CLKDIVN_PDIVN; - - /* todo - set pclk. */ - - /* Write the divisors first with hclk intentionally halved so that - * when we write clkdiv we will under-frequency instead of over. We - * then make a short delay and remove the hclk halving if necessary. - */ - - __raw_writel(camdiv | CAMDIVN_HCLK_HALF, S3C2440_CAMDIVN); - __raw_writel(clkdiv, S3C2410_CLKDIVN); - - ndelay(20); - __raw_writel(camdiv, S3C2440_CAMDIVN); - - clk_set_parent(armclk, cfg->divs.dvs ? hclk : fclk); -} - -static int run_freq_for(unsigned long max_hclk, unsigned long fclk, - int *divs, - struct cpufreq_frequency_table *table, - size_t table_size) -{ - unsigned long freq; - int index = 0; - int div; - - for (div = *divs; div > 0; div = *divs++) { - freq = fclk / div; - - if (freq > max_hclk && div != 1) - continue; - - freq /= 1000; /* table is in kHz */ - index = s3c_cpufreq_addfreq(table, index, table_size, freq); - if (index < 0) - break; - } - - return index; -} - -static int hclk_divs[] = { 1, 2, 3, 4, 6, 8, -1 }; - -static int s3c2440_cpufreq_calctable(struct s3c_cpufreq_config *cfg, - struct cpufreq_frequency_table *table, - size_t table_size) -{ - int ret; - - WARN_ON(cfg->info == NULL); - WARN_ON(cfg->board == NULL); - - ret = run_freq_for(cfg->info->max.hclk, - cfg->info->max.fclk, - hclk_divs, - table, table_size); - - s3c_freq_dbg("%s: returning %d\n", __func__, ret); - - return ret; -} - -struct s3c_cpufreq_info s3c2440_cpufreq_info = { - .max = { - .fclk = 400000000, - .hclk = 133333333, - .pclk = 66666666, - }, - - .locktime_m = 300, - .locktime_u = 300, - .locktime_bits = 16, - - .name = "s3c244x", - .calc_iotiming = s3c2410_iotiming_calc, - .set_iotiming = s3c2410_iotiming_set, - .get_iotiming = s3c2410_iotiming_get, - .set_fvco = s3c2410_set_fvco, - - .set_refresh = s3c2410_cpufreq_setrefresh, - .set_divs = s3c2440_cpufreq_setdivs, - .calc_divs = s3c2440_cpufreq_calcdivs, - .calc_freqtable = s3c2440_cpufreq_calctable, - - .resume_clocks = s3c244x_setup_clocks, - - .debug_io_show = s3c_cpufreq_debugfs_call(s3c2410_iotiming_debugfs), -}; - -static int s3c2440_cpufreq_add(struct device *dev, - struct subsys_interface *sif) -{ - xtal = s3c_cpufreq_clk_get(NULL, "xtal"); - hclk = s3c_cpufreq_clk_get(NULL, "hclk"); - fclk = s3c_cpufreq_clk_get(NULL, "fclk"); - armclk = s3c_cpufreq_clk_get(NULL, "armclk"); - - if (IS_ERR(xtal) || IS_ERR(hclk) || IS_ERR(fclk) || IS_ERR(armclk)) { - printk(KERN_ERR "%s: failed to get clocks\n", __func__); - return -ENOENT; - } - - return s3c_cpufreq_register(&s3c2440_cpufreq_info); -} - -static struct subsys_interface s3c2440_cpufreq_interface = { - .name = "s3c2440_cpufreq", - .subsys = &s3c2440_subsys, - .add_dev = s3c2440_cpufreq_add, -}; - -static int s3c2440_cpufreq_init(void) -{ - return subsys_interface_register(&s3c2440_cpufreq_interface); -} - -/* arch_initcall adds the clocks we need, so use subsys_initcall. */ -subsys_initcall(s3c2440_cpufreq_init); - -static struct subsys_interface s3c2442_cpufreq_interface = { - .name = "s3c2442_cpufreq", - .subsys = &s3c2442_subsys, - .add_dev = s3c2440_cpufreq_add, -}; - -static int s3c2442_cpufreq_init(void) -{ - return subsys_interface_register(&s3c2442_cpufreq_interface); -} -subsys_initcall(s3c2442_cpufreq_init); diff --git a/drivers/cpufreq/s3c24xx-cpufreq-debugfs.c b/drivers/cpufreq/s3c24xx-cpufreq-debugfs.c deleted file mode 100644 index 9b7b4289d66c..000000000000 --- a/drivers/cpufreq/s3c24xx-cpufreq-debugfs.c +++ /dev/null @@ -1,198 +0,0 @@ -/* - * Copyright (c) 2009 Simtec Electronics - * http://armlinux.simtec.co.uk/ - * Ben Dooks <ben@simtec.co.uk> - * - * S3C24XX CPU Frequency scaling - debugfs status support - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. -*/ - -#include <linux/init.h> -#include <linux/export.h> -#include <linux/interrupt.h> -#include <linux/ioport.h> -#include <linux/cpufreq.h> -#include <linux/debugfs.h> -#include <linux/seq_file.h> -#include <linux/err.h> - -#include <plat/cpu-freq-core.h> - -static struct dentry *dbgfs_root; -static struct dentry *dbgfs_file_io; -static struct dentry *dbgfs_file_info; -static struct dentry *dbgfs_file_board; - -#define print_ns(x) ((x) / 10), ((x) % 10) - -static void show_max(struct seq_file *seq, struct s3c_freq *f) -{ - seq_printf(seq, "MAX: F=%lu, H=%lu, P=%lu, A=%lu\n", - f->fclk, f->hclk, f->pclk, f->armclk); -} - -static int board_show(struct seq_file *seq, void *p) -{ - struct s3c_cpufreq_config *cfg; - struct s3c_cpufreq_board *brd; - - cfg = s3c_cpufreq_getconfig(); - if (!cfg) { - seq_printf(seq, "no configuration registered\n"); - return 0; - } - - brd = cfg->board; - if (!brd) { - seq_printf(seq, "no board definition set?\n"); - return 0; - } - - seq_printf(seq, "SDRAM refresh %u ns\n", brd->refresh); - seq_printf(seq, "auto_io=%u\n", brd->auto_io); - seq_printf(seq, "need_io=%u\n", brd->need_io); - - show_max(seq, &brd->max); - - - return 0; -} - -static int fops_board_open(struct inode *inode, struct file *file) -{ - return single_open(file, board_show, NULL); -} - -static const struct file_operations fops_board = { - .open = fops_board_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, - .owner = THIS_MODULE, -}; - -static int info_show(struct seq_file *seq, void *p) -{ - struct s3c_cpufreq_config *cfg; - - cfg = s3c_cpufreq_getconfig(); - if (!cfg) { - seq_printf(seq, "no configuration registered\n"); - return 0; - } - - seq_printf(seq, " FCLK %ld Hz\n", cfg->freq.fclk); - seq_printf(seq, " HCLK %ld Hz (%lu.%lu ns)\n", - cfg->freq.hclk, print_ns(cfg->freq.hclk_tns)); - seq_printf(seq, " PCLK %ld Hz\n", cfg->freq.hclk); - seq_printf(seq, "ARMCLK %ld Hz\n", cfg->freq.armclk); - seq_printf(seq, "\n"); - - show_max(seq, &cfg->max); - - seq_printf(seq, "Divisors: P=%d, H=%d, A=%d, dvs=%s\n", - cfg->divs.h_divisor, cfg->divs.p_divisor, - cfg->divs.arm_divisor, cfg->divs.dvs ? "on" : "off"); - seq_printf(seq, "\n"); - - seq_printf(seq, "lock_pll=%u\n", cfg->lock_pll); - - return 0; -} - -static int fops_info_open(struct inode *inode, struct file *file) -{ - return single_open(file, info_show, NULL); -} - -static const struct file_operations fops_info = { - .open = fops_info_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, - .owner = THIS_MODULE, -}; - -static int io_show(struct seq_file *seq, void *p) -{ - void (*show_bank)(struct seq_file *, struct s3c_cpufreq_config *, union s3c_iobank *); - struct s3c_cpufreq_config *cfg; - struct s3c_iotimings *iot; - union s3c_iobank *iob; - int bank; - - cfg = s3c_cpufreq_getconfig(); - if (!cfg) { - seq_printf(seq, "no configuration registered\n"); - return 0; - } - - show_bank = cfg->info->debug_io_show; - if (!show_bank) { - seq_printf(seq, "no code to show bank timing\n"); - return 0; - } - - iot = s3c_cpufreq_getiotimings(); - if (!iot) { - seq_printf(seq, "no io timings registered\n"); - return 0; - } - - seq_printf(seq, "hclk period is %lu.%lu ns\n", print_ns(cfg->freq.hclk_tns)); - - for (bank = 0; bank < MAX_BANKS; bank++) { - iob = &iot->bank[bank]; - - seq_printf(seq, "bank %d: ", bank); - - if (!iob->io_2410) { - seq_printf(seq, "nothing set\n"); - continue; - } - - show_bank(seq, cfg, iob); - } - - return 0; -} - -static int fops_io_open(struct inode *inode, struct file *file) -{ - return single_open(file, io_show, NULL); -} - -static const struct file_operations fops_io = { - .open = fops_io_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, - .owner = THIS_MODULE, -}; - - -static int __init s3c_freq_debugfs_init(void) -{ - dbgfs_root = debugfs_create_dir("s3c-cpufreq", NULL); - if (IS_ERR(dbgfs_root)) { - printk(KERN_ERR "%s: error creating debugfs root\n", __func__); - return PTR_ERR(dbgfs_root); - } - - dbgfs_file_io = debugfs_create_file("io-timing", S_IRUGO, dbgfs_root, - NULL, &fops_io); - - dbgfs_file_info = debugfs_create_file("info", S_IRUGO, dbgfs_root, - NULL, &fops_info); - - dbgfs_file_board = debugfs_create_file("board", S_IRUGO, dbgfs_root, - NULL, &fops_board); - - return 0; -} - -late_initcall(s3c_freq_debugfs_init); - diff --git a/drivers/cpufreq/s3c24xx-cpufreq.c b/drivers/cpufreq/s3c24xx-cpufreq.c deleted file mode 100644 index 3513e7477160..000000000000 --- a/drivers/cpufreq/s3c24xx-cpufreq.c +++ /dev/null @@ -1,711 +0,0 @@ -/* - * Copyright (c) 2006-2008 Simtec Electronics - * http://armlinux.simtec.co.uk/ - * Ben Dooks <ben@simtec.co.uk> - * - * S3C24XX CPU Frequency scaling - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. -*/ - -#include <linux/init.h> -#include <linux/module.h> -#include <linux/interrupt.h> -#include <linux/ioport.h> -#include <linux/cpufreq.h> -#include <linux/cpu.h> -#include <linux/clk.h> -#include <linux/err.h> -#include <linux/io.h> -#include <linux/device.h> -#include <linux/sysfs.h> -#include <linux/slab.h> - -#include <asm/mach/arch.h> -#include <asm/mach/map.h> - -#include <plat/cpu.h> -#include <plat/clock.h> -#include <plat/cpu-freq-core.h> - -#include <mach/regs-clock.h> - -/* note, cpufreq support deals in kHz, no Hz */ - -static struct cpufreq_driver s3c24xx_driver; -static struct s3c_cpufreq_config cpu_cur; -static struct s3c_iotimings s3c24xx_iotiming; -static struct cpufreq_frequency_table *pll_reg; -static unsigned int last_target = ~0; -static unsigned int ftab_size; -static struct cpufreq_frequency_table *ftab; - -static struct clk *_clk_mpll; -static struct clk *_clk_xtal; -static struct clk *clk_fclk; -static struct clk *clk_hclk; -static struct clk *clk_pclk; -static struct clk *clk_arm; - -#ifdef CONFIG_CPU_FREQ_S3C24XX_DEBUGFS -struct s3c_cpufreq_config *s3c_cpufreq_getconfig(void) -{ - return &cpu_cur; -} - -struct s3c_iotimings *s3c_cpufreq_getiotimings(void) -{ - return &s3c24xx_iotiming; -} -#endif /* CONFIG_CPU_FREQ_S3C24XX_DEBUGFS */ - -static void s3c_cpufreq_getcur(struct s3c_cpufreq_config *cfg) -{ - unsigned long fclk, pclk, hclk, armclk; - - cfg->freq.fclk = fclk = clk_get_rate(clk_fclk); - cfg->freq.hclk = hclk = clk_get_rate(clk_hclk); - cfg->freq.pclk = pclk = clk_get_rate(clk_pclk); - cfg->freq.armclk = armclk = clk_get_rate(clk_arm); - - cfg->pll.driver_data = __raw_readl(S3C2410_MPLLCON); - cfg->pll.frequency = fclk; - - cfg->freq.hclk_tns = 1000000000 / (cfg->freq.hclk / 10); - - cfg->divs.h_divisor = fclk / hclk; - cfg->divs.p_divisor = fclk / pclk; -} - -static inline void s3c_cpufreq_calc(struct s3c_cpufreq_config *cfg) -{ - unsigned long pll = cfg->pll.frequency; - - cfg->freq.fclk = pll; - cfg->freq.hclk = pll / cfg->divs.h_divisor; - cfg->freq.pclk = pll / cfg->divs.p_divisor; - - /* convert hclk into 10ths of nanoseconds for io calcs */ - cfg->freq.hclk_tns = 1000000000 / (cfg->freq.hclk / 10); -} - -static inline int closer(unsigned int target, unsigned int n, unsigned int c) -{ - int diff_cur = abs(target - c); - int diff_new = abs(target - n); - - return (diff_new < diff_cur); -} - -static void s3c_cpufreq_show(const char *pfx, - struct s3c_cpufreq_config *cfg) -{ - s3c_freq_dbg("%s: Fvco=%u, F=%lu, A=%lu, H=%lu (%u), P=%lu (%u)\n", - pfx, cfg->pll.frequency, cfg->freq.fclk, cfg->freq.armclk, - cfg->freq.hclk, cfg->divs.h_divisor, - cfg->freq.pclk, cfg->divs.p_divisor); -} - -/* functions to wrapper the driver info calls to do the cpu specific work */ - -static void s3c_cpufreq_setio(struct s3c_cpufreq_config *cfg) -{ - if (cfg->info->set_iotiming) - (cfg->info->set_iotiming)(cfg, &s3c24xx_iotiming); -} - -static int s3c_cpufreq_calcio(struct s3c_cpufreq_config *cfg) -{ - if (cfg->info->calc_iotiming) - return (cfg->info->calc_iotiming)(cfg, &s3c24xx_iotiming); - - return 0; -} - -static void s3c_cpufreq_setrefresh(struct s3c_cpufreq_config *cfg) -{ - (cfg->info->set_refresh)(cfg); -} - -static void s3c_cpufreq_setdivs(struct s3c_cpufreq_config *cfg) -{ - (cfg->info->set_divs)(cfg); -} - -static int s3c_cpufreq_calcdivs(struct s3c_cpufreq_config *cfg) -{ - return (cfg->info->calc_divs)(cfg); -} - -static void s3c_cpufreq_setfvco(struct s3c_cpufreq_config *cfg) -{ - (cfg->info->set_fvco)(cfg); -} - -static inline void s3c_cpufreq_resume_clocks(void) -{ - cpu_cur.info->resume_clocks(); -} - -static inline void s3c_cpufreq_updateclk(struct clk *clk, - unsigned int freq) -{ - clk_set_rate(clk, freq); -} - -static int s3c_cpufreq_settarget(struct cpufreq_policy *policy, - unsigned int target_freq, - struct cpufreq_frequency_table *pll) -{ - struct s3c_cpufreq_freqs freqs; - struct s3c_cpufreq_config cpu_new; - unsigned long flags; - - cpu_new = cpu_cur; /* copy new from current */ - - s3c_cpufreq_show("cur", &cpu_cur); - - /* TODO - check for DMA currently outstanding */ - - cpu_new.pll = pll ? *pll : cpu_cur.pll; - - if (pll) - freqs.pll_changing = 1; - - /* update our frequencies */ - - cpu_new.freq.armclk = target_freq; - cpu_new.freq.fclk = cpu_new.pll.frequency; - - if (s3c_cpufreq_calcdivs(&cpu_new) < 0) { - printk(KERN_ERR "no divisors for %d\n", target_freq); - goto err_notpossible; - } - - s3c_freq_dbg("%s: got divs\n", __func__); - - s3c_cpufreq_calc(&cpu_new); - - s3c_freq_dbg("%s: calculated frequencies for new\n", __func__); - - if (cpu_new.freq.hclk != cpu_cur.freq.hclk) { - if (s3c_cpufreq_calcio(&cpu_new) < 0) { - printk(KERN_ERR "%s: no IO timings\n", __func__); - goto err_notpossible; - } - } - - s3c_cpufreq_show("new", &cpu_new); - - /* setup our cpufreq parameters */ - - freqs.old = cpu_cur.freq; - freqs.new = cpu_new.freq; - - freqs.freqs.old = cpu_cur.freq.armclk / 1000; - freqs.freqs.new = cpu_new.freq.armclk / 1000; - - /* update f/h/p clock settings before we issue the change - * notification, so that drivers do not need to do anything - * special if they want to recalculate on CPUFREQ_PRECHANGE. */ - - s3c_cpufreq_updateclk(_clk_mpll, cpu_new.pll.frequency); - s3c_cpufreq_updateclk(clk_fclk, cpu_new.freq.fclk); - s3c_cpufreq_updateclk(clk_hclk, cpu_new.freq.hclk); - s3c_cpufreq_updateclk(clk_pclk, cpu_new.freq.pclk); - - /* start the frequency change */ - cpufreq_notify_transition(policy, &freqs.freqs, CPUFREQ_PRECHANGE); - - /* If hclk is staying the same, then we do not need to - * re-write the IO or the refresh timings whilst we are changing - * speed. */ - - local_irq_save(flags); - - /* is our memory clock slowing down? */ - if (cpu_new.freq.hclk < cpu_cur.freq.hclk) { - s3c_cpufreq_setrefresh(&cpu_new); - s3c_cpufreq_setio(&cpu_new); - } - - if (cpu_new.freq.fclk == cpu_cur.freq.fclk) { - /* not changing PLL, just set the divisors */ - - s3c_cpufreq_setdivs(&cpu_new); - } else { - if (cpu_new.freq.fclk < cpu_cur.freq.fclk) { - /* slow the cpu down, then set divisors */ - - s3c_cpufreq_setfvco(&cpu_new); - s3c_cpufreq_setdivs(&cpu_new); - } else { - /* set the divisors, then speed up */ - - s3c_cpufreq_setdivs(&cpu_new); - s3c_cpufreq_setfvco(&cpu_new); - } - } - - /* did our memory clock speed up */ - if (cpu_new.freq.hclk > cpu_cur.freq.hclk) { - s3c_cpufreq_setrefresh(&cpu_new); - s3c_cpufreq_setio(&cpu_new); - } - - /* update our current settings */ - cpu_cur = cpu_new; - - local_irq_restore(flags); - - /* notify everyone we've done this */ - cpufreq_notify_transition(policy, &freqs.freqs, CPUFREQ_POSTCHANGE); - - s3c_freq_dbg("%s: finished\n", __func__); - return 0; - - err_notpossible: - printk(KERN_ERR "no compatible settings for %d\n", target_freq); - return -EINVAL; -} - -/* s3c_cpufreq_target - * - * called by the cpufreq core to adjust the frequency that the CPU - * is currently running at. - */ - -static int s3c_cpufreq_target(struct cpufreq_policy *policy, - unsigned int target_freq, - unsigned int relation) -{ - struct cpufreq_frequency_table *pll; - unsigned int index; - - /* avoid repeated calls which cause a needless amout of duplicated - * logging output (and CPU time as the calculation process is - * done) */ - if (target_freq == last_target) - return 0; - - last_target = target_freq; - - s3c_freq_dbg("%s: policy %p, target %u, relation %u\n", - __func__, policy, target_freq, relation); - - if (ftab) { - if (cpufreq_frequency_table_target(policy, ftab, - target_freq, relation, - &index)) { - s3c_freq_dbg("%s: table failed\n", __func__); - return -EINVAL; - } - - s3c_freq_dbg("%s: adjust %d to entry %d (%u)\n", __func__, - target_freq, index, ftab[index].frequency); - target_freq = ftab[index].frequency; - } - - target_freq *= 1000; /* convert target to Hz */ - - /* find the settings for our new frequency */ - - if (!pll_reg || cpu_cur.lock_pll) { - /* either we've not got any PLL values, or we've locked - * to the current one. */ - pll = NULL; - } else { - struct cpufreq_policy tmp_policy; - int ret; - - /* we keep the cpu pll table in Hz, to ensure we get an - * accurate value for the PLL output. */ - - tmp_policy.min = policy->min * 1000; - tmp_policy.max = policy->max * 1000; - tmp_policy.cpu = policy->cpu; - - /* cpufreq_frequency_table_target uses a pointer to 'index' - * which is the number of the table entry, not the value of - * the table entry's index field. */ - - ret = cpufreq_frequency_table_target(&tmp_policy, pll_reg, - target_freq, relation, - &index); - - if (ret < 0) { - printk(KERN_ERR "%s: no PLL available\n", __func__); - goto err_notpossible; - } - - pll = pll_reg + index; - - s3c_freq_dbg("%s: target %u => %u\n", - __func__, target_freq, pll->frequency); - - target_freq = pll->frequency; - } - - return s3c_cpufreq_settarget(policy, target_freq, pll); - - err_notpossible: - printk(KERN_ERR "no compatible settings for %d\n", target_freq); - return -EINVAL; -} - -static unsigned int s3c_cpufreq_get(unsigned int cpu) -{ - return clk_get_rate(clk_arm) / 1000; -} - -struct clk *s3c_cpufreq_clk_get(struct device *dev, const char *name) -{ - struct clk *clk; - - clk = clk_get(dev, name); - if (IS_ERR(clk)) - printk(KERN_ERR "cpufreq: failed to get clock '%s'\n", name); - - return clk; -} - -static int s3c_cpufreq_init(struct cpufreq_policy *policy) -{ - printk(KERN_INFO "%s: initialising policy %p\n", __func__, policy); - - if (policy->cpu != 0) - return -EINVAL; - - policy->cur = s3c_cpufreq_get(0); - policy->min = policy->cpuinfo.min_freq = 0; - policy->max = policy->cpuinfo.max_freq = cpu_cur.info->max.fclk / 1000; - policy->governor = CPUFREQ_DEFAULT_GOVERNOR; - - /* feed the latency information from the cpu driver */ - policy->cpuinfo.transition_latency = cpu_cur.info->latency; - - if (ftab) - cpufreq_frequency_table_cpuinfo(policy, ftab); - - return 0; -} - -static __init int s3c_cpufreq_initclks(void) -{ - _clk_mpll = s3c_cpufreq_clk_get(NULL, "mpll"); - _clk_xtal = s3c_cpufreq_clk_get(NULL, "xtal"); - clk_fclk = s3c_cpufreq_clk_get(NULL, "fclk"); - clk_hclk = s3c_cpufreq_clk_get(NULL, "hclk"); - clk_pclk = s3c_cpufreq_clk_get(NULL, "pclk"); - clk_arm = s3c_cpufreq_clk_get(NULL, "armclk"); - - if (IS_ERR(clk_fclk) || IS_ERR(clk_hclk) || IS_ERR(clk_pclk) || - IS_ERR(_clk_mpll) || IS_ERR(clk_arm) || IS_ERR(_clk_xtal)) { - printk(KERN_ERR "%s: could not get clock(s)\n", __func__); - return -ENOENT; - } - - printk(KERN_INFO "%s: clocks f=%lu,h=%lu,p=%lu,a=%lu\n", __func__, - clk_get_rate(clk_fclk) / 1000, - clk_get_rate(clk_hclk) / 1000, - clk_get_rate(clk_pclk) / 1000, - clk_get_rate(clk_arm) / 1000); - - return 0; -} - -static int s3c_cpufreq_verify(struct cpufreq_policy *policy) -{ - if (policy->cpu != 0) - return -EINVAL; - - return 0; -} - -#ifdef CONFIG_PM -static struct cpufreq_frequency_table suspend_pll; -static unsigned int suspend_freq; - -static int s3c_cpufreq_suspend(struct cpufreq_policy *policy) -{ - suspend_pll.frequency = clk_get_rate(_clk_mpll); - suspend_pll.driver_data = __raw_readl(S3C2410_MPLLCON); - suspend_freq = s3c_cpufreq_get(0) * 1000; - - return 0; -} - -static int s3c_cpufreq_resume(struct cpufreq_policy *policy) -{ - int ret; - - s3c_freq_dbg("%s: resuming with policy %p\n", __func__, policy); - - last_target = ~0; /* invalidate last_target setting */ - - /* first, find out what speed we resumed at. */ - s3c_cpufreq_resume_clocks(); - - /* whilst we will be called later on, we try and re-set the - * cpu frequencies as soon as possible so that we do not end - * up resuming devices and then immediately having to re-set - * a number of settings once these devices have restarted. - * - * as a note, it is expected devices are not used until they - * have been un-suspended and at that time they should have - * used the updated clock settings. - */ - - ret = s3c_cpufreq_settarget(NULL, suspend_freq, &suspend_pll); - if (ret) { - printk(KERN_ERR "%s: failed to reset pll/freq\n", __func__); - return ret; - } - - return 0; -} -#else -#define s3c_cpufreq_resume NULL -#define s3c_cpufreq_suspend NULL -#endif - -static struct cpufreq_driver s3c24xx_driver = { - .flags = CPUFREQ_STICKY, - .verify = s3c_cpufreq_verify, - .target = s3c_cpufreq_target, - .get = s3c_cpufreq_get, - .init = s3c_cpufreq_init, - .suspend = s3c_cpufreq_suspend, - .resume = s3c_cpufreq_resume, - .name = "s3c24xx", -}; - - -int __init s3c_cpufreq_register(struct s3c_cpufreq_info *info) -{ - if (!info || !info->name) { - printk(KERN_ERR "%s: failed to pass valid information\n", - __func__); - return -EINVAL; - } - - printk(KERN_INFO "S3C24XX CPU Frequency driver, %s cpu support\n", - info->name); - - /* check our driver info has valid data */ - - BUG_ON(info->set_refresh == NULL); - BUG_ON(info->set_divs == NULL); - BUG_ON(info->calc_divs == NULL); - - /* info->set_fvco is optional, depending on whether there - * is a need to set the clock code. */ - - cpu_cur.info = info; - - /* Note, driver registering should probably update locktime */ - - return 0; -} - -int __init s3c_cpufreq_setboard(struct s3c_cpufreq_board *board) -{ - struct s3c_cpufreq_board *ours; - - if (!board) { - printk(KERN_INFO "%s: no board data\n", __func__); - return -EINVAL; - } - - /* Copy the board information so that each board can make this - * initdata. */ - - ours = kzalloc(sizeof(struct s3c_cpufreq_board), GFP_KERNEL); - if (ours == NULL) { - printk(KERN_ERR "%s: no memory\n", __func__); - return -ENOMEM; - } - - *ours = *board; - cpu_cur.board = ours; - - return 0; -} - -int __init s3c_cpufreq_auto_io(void) -{ - int ret; - - if (!cpu_cur.info->get_iotiming) { - printk(KERN_ERR "%s: get_iotiming undefined\n", __func__); - return -ENOENT; - } - - printk(KERN_INFO "%s: working out IO settings\n", __func__); - - ret = (cpu_cur.info->get_iotiming)(&cpu_cur, &s3c24xx_iotiming); - if (ret) - printk(KERN_ERR "%s: failed to get timings\n", __func__); - - return ret; -} - -/* if one or is zero, then return the other, otherwise return the min */ -#define do_min(_a, _b) ((_a) == 0 ? (_b) : (_b) == 0 ? (_a) : min(_a, _b)) - -/** - * s3c_cpufreq_freq_min - find the minimum settings for the given freq. - * @dst: The destination structure - * @a: One argument. - * @b: The other argument. - * - * Create a minimum of each frequency entry in the 'struct s3c_freq', - * unless the entry is zero when it is ignored and the non-zero argument - * used. - */ -static void s3c_cpufreq_freq_min(struct s3c_freq *dst, - struct s3c_freq *a, struct s3c_freq *b) -{ - dst->fclk = do_min(a->fclk, b->fclk); - dst->hclk = do_min(a->hclk, b->hclk); - dst->pclk = do_min(a->pclk, b->pclk); - dst->armclk = do_min(a->armclk, b->armclk); -} - -static inline u32 calc_locktime(u32 freq, u32 time_us) -{ - u32 result; - - result = freq * time_us; - result = DIV_ROUND_UP(result, 1000 * 1000); - - return result; -} - -static void s3c_cpufreq_update_loctkime(void) -{ - unsigned int bits = cpu_cur.info->locktime_bits; - u32 rate = (u32)clk_get_rate(_clk_xtal); - u32 val; - - if (bits == 0) { - WARN_ON(1); - return; - } - - val = calc_locktime(rate, cpu_cur.info->locktime_u) << bits; - val |= calc_locktime(rate, cpu_cur.info->locktime_m); - - printk(KERN_INFO "%s: new locktime is 0x%08x\n", __func__, val); - __raw_writel(val, S3C2410_LOCKTIME); -} - -static int s3c_cpufreq_build_freq(void) -{ - int size, ret; - - if (!cpu_cur.info->calc_freqtable) - return -EINVAL; - - kfree(ftab); - ftab = NULL; - - size = cpu_cur.info->calc_freqtable(&cpu_cur, NULL, 0); - size++; - - ftab = kmalloc(sizeof(struct cpufreq_frequency_table) * size, GFP_KERNEL); - if (!ftab) { - printk(KERN_ERR "%s: no memory for tables\n", __func__); - return -ENOMEM; - } - - ftab_size = size; - - ret = cpu_cur.info->calc_freqtable(&cpu_cur, ftab, size); - s3c_cpufreq_addfreq(ftab, ret, size, CPUFREQ_TABLE_END); - - return 0; -} - -static int __init s3c_cpufreq_initcall(void) -{ - int ret = 0; - - if (cpu_cur.info && cpu_cur.board) { - ret = s3c_cpufreq_initclks(); - if (ret) - goto out; - - /* get current settings */ - s3c_cpufreq_getcur(&cpu_cur); - s3c_cpufreq_show("cur", &cpu_cur); - - if (cpu_cur.board->auto_io) { - ret = s3c_cpufreq_auto_io(); - if (ret) { - printk(KERN_ERR "%s: failed to get io timing\n", - __func__); - goto out; - } - } - - if (cpu_cur.board->need_io && !cpu_cur.info->set_iotiming) { - printk(KERN_ERR "%s: no IO support registered\n", - __func__); - ret = -EINVAL; - goto out; - } - - if (!cpu_cur.info->need_pll) - cpu_cur.lock_pll = 1; - - s3c_cpufreq_update_loctkime(); - - s3c_cpufreq_freq_min(&cpu_cur.max, &cpu_cur.board->max, - &cpu_cur.info->max); - - if (cpu_cur.info->calc_freqtable) - s3c_cpufreq_build_freq(); - - ret = cpufreq_register_driver(&s3c24xx_driver); - } - - out: - return ret; -} - -late_initcall(s3c_cpufreq_initcall); - -/** - * s3c_plltab_register - register CPU PLL table. - * @plls: The list of PLL entries. - * @plls_no: The size of the PLL entries @plls. - * - * Register the given set of PLLs with the system. - */ -int __init s3c_plltab_register(struct cpufreq_frequency_table *plls, - unsigned int plls_no) -{ - struct cpufreq_frequency_table *vals; - unsigned int size; - - size = sizeof(struct cpufreq_frequency_table) * (plls_no + 1); - - vals = kmalloc(size, GFP_KERNEL); - if (vals) { - memcpy(vals, plls, size); - pll_reg = vals; - - /* write a terminating entry, we don't store it in the - * table that is stored in the kernel */ - vals += plls_no; - vals->frequency = CPUFREQ_TABLE_END; - - printk(KERN_INFO "cpufreq: %d PLL entries\n", plls_no); - } else - printk(KERN_ERR "cpufreq: no memory for PLL tables\n"); - - return vals ? 0 : -ENOMEM; -} diff --git a/drivers/cpufreq/s3c64xx-cpufreq.c b/drivers/cpufreq/s3c64xx-cpufreq.c index 13bb4bae64ee..9cef71528076 100644 --- a/drivers/cpufreq/s3c64xx-cpufreq.c +++ b/drivers/cpufreq/s3c64xx-cpufreq.c @@ -1,11 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Copyright 2009 Wolfson Microelectronics plc * * S3C64xx CPUfreq Support - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #define pr_fmt(fmt) "cpufreq: " fmt @@ -19,16 +16,15 @@ #include <linux/regulator/consumer.h> #include <linux/module.h> -static struct clk *armclk; static struct regulator *vddarm; static unsigned long regulator_latency; -#ifdef CONFIG_CPU_S3C6410 struct s3c64xx_dvfs { unsigned int vddarm_min; unsigned int vddarm_max; }; +#ifdef CONFIG_REGULATOR static struct s3c64xx_dvfs s3c64xx_dvfs_table[] = { [0] = { 1000000, 1150000 }, [1] = { 1050000, 1150000 }, @@ -36,121 +32,80 @@ static struct s3c64xx_dvfs s3c64xx_dvfs_table[] = { [3] = { 1200000, 1350000 }, [4] = { 1300000, 1350000 }, }; +#endif static struct cpufreq_frequency_table s3c64xx_freq_table[] = { - { 0, 66000 }, - { 0, 100000 }, - { 0, 133000 }, - { 1, 200000 }, - { 1, 222000 }, - { 1, 266000 }, - { 2, 333000 }, - { 2, 400000 }, - { 2, 532000 }, - { 2, 533000 }, - { 3, 667000 }, - { 4, 800000 }, - { 0, CPUFREQ_TABLE_END }, + { 0, 0, 66000 }, + { 0, 0, 100000 }, + { 0, 0, 133000 }, + { 0, 1, 200000 }, + { 0, 1, 222000 }, + { 0, 1, 266000 }, + { 0, 2, 333000 }, + { 0, 2, 400000 }, + { 0, 2, 532000 }, + { 0, 2, 533000 }, + { 0, 3, 667000 }, + { 0, 4, 800000 }, + { 0, 0, CPUFREQ_TABLE_END }, }; -#endif - -static int s3c64xx_cpufreq_verify_speed(struct cpufreq_policy *policy) -{ - if (policy->cpu != 0) - return -EINVAL; - - return cpufreq_frequency_table_verify(policy, s3c64xx_freq_table); -} - -static unsigned int s3c64xx_cpufreq_get_speed(unsigned int cpu) -{ - if (cpu != 0) - return 0; - - return clk_get_rate(armclk) / 1000; -} static int s3c64xx_cpufreq_set_target(struct cpufreq_policy *policy, - unsigned int target_freq, - unsigned int relation) + unsigned int index) { + unsigned int new_freq = s3c64xx_freq_table[index].frequency; int ret; - unsigned int i; - struct cpufreq_freqs freqs; - struct s3c64xx_dvfs *dvfs; - - ret = cpufreq_frequency_table_target(policy, s3c64xx_freq_table, - target_freq, relation, &i); - if (ret != 0) - return ret; - - freqs.old = clk_get_rate(armclk) / 1000; - freqs.new = s3c64xx_freq_table[i].frequency; - freqs.flags = 0; - dvfs = &s3c64xx_dvfs_table[s3c64xx_freq_table[i].driver_data]; - if (freqs.old == freqs.new) - return 0; - - pr_debug("Transition %d-%dkHz\n", freqs.old, freqs.new); +#ifdef CONFIG_REGULATOR + struct s3c64xx_dvfs *dvfs; + unsigned int old_freq; - cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE); + old_freq = clk_get_rate(policy->clk) / 1000; + dvfs = &s3c64xx_dvfs_table[s3c64xx_freq_table[index].driver_data]; -#ifdef CONFIG_REGULATOR - if (vddarm && freqs.new > freqs.old) { + if (vddarm && new_freq > old_freq) { ret = regulator_set_voltage(vddarm, dvfs->vddarm_min, dvfs->vddarm_max); if (ret != 0) { pr_err("Failed to set VDDARM for %dkHz: %d\n", - freqs.new, ret); - freqs.new = freqs.old; - goto post_notify; + new_freq, ret); + return ret; } } #endif - ret = clk_set_rate(armclk, freqs.new * 1000); + ret = clk_set_rate(policy->clk, new_freq * 1000); if (ret < 0) { pr_err("Failed to set rate %dkHz: %d\n", - freqs.new, ret); - freqs.new = freqs.old; + new_freq, ret); + return ret; } -post_notify: - cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); - if (ret) - goto err; - #ifdef CONFIG_REGULATOR - if (vddarm && freqs.new < freqs.old) { + if (vddarm && new_freq < old_freq) { ret = regulator_set_voltage(vddarm, dvfs->vddarm_min, dvfs->vddarm_max); if (ret != 0) { pr_err("Failed to set VDDARM for %dkHz: %d\n", - freqs.new, ret); - goto err_clk; + new_freq, ret); + if (clk_set_rate(policy->clk, old_freq * 1000) < 0) + pr_err("Failed to restore original clock rate\n"); + + return ret; } } #endif pr_debug("Set actual frequency %lukHz\n", - clk_get_rate(armclk) / 1000); + clk_get_rate(policy->clk) / 1000); return 0; - -err_clk: - if (clk_set_rate(armclk, freqs.old * 1000) < 0) - pr_err("Failed to restore original clock rate\n"); -err: - cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); - - return ret; } #ifdef CONFIG_REGULATOR -static void __init s3c64xx_cpufreq_config_regulator(void) +static void s3c64xx_cpufreq_config_regulator(void) { int count, v, i, found; struct cpufreq_frequency_table *freq; @@ -161,12 +116,11 @@ static void __init s3c64xx_cpufreq_config_regulator(void) pr_err("Unable to check supported voltages\n"); } - freq = s3c64xx_freq_table; - while (count > 0 && freq->frequency != CPUFREQ_TABLE_END) { - if (freq->frequency == CPUFREQ_ENTRY_INVALID) - continue; + if (!count) + goto out; - dvfs = &s3c64xx_dvfs_table[freq->index]; + cpufreq_for_each_valid_entry(freq, s3c64xx_freq_table) { + dvfs = &s3c64xx_dvfs_table[freq->driver_data]; found = 0; for (i = 0; i < count; i++) { @@ -180,10 +134,9 @@ static void __init s3c64xx_cpufreq_config_regulator(void) freq->frequency); freq->frequency = CPUFREQ_ENTRY_INVALID; } - - freq++; } +out: /* Guess based on having to do an I2C/SPI write; in future we * will be able to query the regulator performance here. */ regulator_latency = 1 * 1000 * 1000; @@ -192,29 +145,22 @@ static void __init s3c64xx_cpufreq_config_regulator(void) static int s3c64xx_cpufreq_driver_init(struct cpufreq_policy *policy) { - int ret; struct cpufreq_frequency_table *freq; if (policy->cpu != 0) return -EINVAL; - if (s3c64xx_freq_table == NULL) { - pr_err("No frequency information for this CPU\n"); - return -ENODEV; - } - - armclk = clk_get(NULL, "armclk"); - if (IS_ERR(armclk)) { + policy->clk = clk_get(NULL, "armclk"); + if (IS_ERR(policy->clk)) { pr_err("Unable to obtain ARMCLK: %ld\n", - PTR_ERR(armclk)); - return PTR_ERR(armclk); + PTR_ERR(policy->clk)); + return PTR_ERR(policy->clk); } #ifdef CONFIG_REGULATOR vddarm = regulator_get(NULL, "vddarm"); if (IS_ERR(vddarm)) { - ret = PTR_ERR(vddarm); - pr_err("Failed to obtain VDDARM: %d\n", ret); + pr_err("Failed to obtain VDDARM: %ld\n", PTR_ERR(vddarm)); pr_err("Only frequency scaling available\n"); vddarm = NULL; } else { @@ -222,12 +168,11 @@ static int s3c64xx_cpufreq_driver_init(struct cpufreq_policy *policy) } #endif - freq = s3c64xx_freq_table; - while (freq->frequency != CPUFREQ_TABLE_END) { + cpufreq_for_each_entry(freq, s3c64xx_freq_table) { unsigned long r; /* Check for frequencies we can generate */ - r = clk_round_rate(armclk, freq->frequency * 1000); + r = clk_round_rate(policy->clk, freq->frequency * 1000); r /= 1000; if (r != freq->frequency) { pr_debug("%dkHz unsupported by clock\n", @@ -237,37 +182,24 @@ static int s3c64xx_cpufreq_driver_init(struct cpufreq_policy *policy) /* If we have no regulator then assume startup * frequency is the maximum we can support. */ - if (!vddarm && freq->frequency > s3c64xx_cpufreq_get_speed(0)) + if (!vddarm && freq->frequency > clk_get_rate(policy->clk) / 1000) freq->frequency = CPUFREQ_ENTRY_INVALID; - - freq++; } - policy->cur = clk_get_rate(armclk) / 1000; - /* Datasheet says PLL stabalisation time (if we were to use * the PLLs, which we don't currently) is ~300us worst case, * but add some fudge. */ - policy->cpuinfo.transition_latency = (500 * 1000) + regulator_latency; - - ret = cpufreq_frequency_table_cpuinfo(policy, s3c64xx_freq_table); - if (ret != 0) { - pr_err("Failed to configure frequency table: %d\n", - ret); - regulator_put(vddarm); - clk_put(armclk); - } - - return ret; + cpufreq_generic_init(policy, s3c64xx_freq_table, + (500 * 1000) + regulator_latency); + return 0; } static struct cpufreq_driver s3c64xx_cpufreq_driver = { - .owner = THIS_MODULE, - .flags = 0, - .verify = s3c64xx_cpufreq_verify_speed, - .target = s3c64xx_cpufreq_set_target, - .get = s3c64xx_cpufreq_get_speed, + .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK, + .verify = cpufreq_generic_frequency_table_verify, + .target_index = s3c64xx_cpufreq_set_target, + .get = cpufreq_generic_get, .init = s3c64xx_cpufreq_driver_init, .name = "s3c", }; diff --git a/drivers/cpufreq/s5pv210-cpufreq.c b/drivers/cpufreq/s5pv210-cpufreq.c index 5c7757073793..ba8a1c96427a 100644 --- a/drivers/cpufreq/s5pv210-cpufreq.c +++ b/drivers/cpufreq/s5pv210-cpufreq.c @@ -1,14 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (c) 2010 Samsung Electronics Co., Ltd. * http://www.samsung.com * * CPU frequency scaling for S5PC110/S5PV210 - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + #include <linux/types.h> #include <linux/kernel.h> #include <linux/init.h> @@ -16,17 +15,73 @@ #include <linux/clk.h> #include <linux/io.h> #include <linux/cpufreq.h> +#include <linux/of.h> +#include <linux/of_address.h> +#include <linux/platform_device.h> #include <linux/reboot.h> #include <linux/regulator/consumer.h> -#include <linux/suspend.h> -#include <mach/map.h> -#include <mach/regs-clock.h> +static void __iomem *clk_base; +static void __iomem *dmc_base[2]; + +#define S5P_CLKREG(x) (clk_base + (x)) + +#define S5P_APLL_LOCK S5P_CLKREG(0x00) +#define S5P_APLL_CON S5P_CLKREG(0x100) +#define S5P_CLK_SRC0 S5P_CLKREG(0x200) +#define S5P_CLK_SRC2 S5P_CLKREG(0x208) +#define S5P_CLK_DIV0 S5P_CLKREG(0x300) +#define S5P_CLK_DIV2 S5P_CLKREG(0x308) +#define S5P_CLK_DIV6 S5P_CLKREG(0x318) +#define S5P_CLKDIV_STAT0 S5P_CLKREG(0x1000) +#define S5P_CLKDIV_STAT1 S5P_CLKREG(0x1004) +#define S5P_CLKMUX_STAT0 S5P_CLKREG(0x1100) +#define S5P_CLKMUX_STAT1 S5P_CLKREG(0x1104) + +#define S5P_ARM_MCS_CON S5P_CLKREG(0x6100) + +/* CLKSRC0 */ +#define S5P_CLKSRC0_MUX200_SHIFT (16) +#define S5P_CLKSRC0_MUX200_MASK (0x1 << S5P_CLKSRC0_MUX200_SHIFT) +#define S5P_CLKSRC0_MUX166_MASK (0x1<<20) +#define S5P_CLKSRC0_MUX133_MASK (0x1<<24) + +/* CLKSRC2 */ +#define S5P_CLKSRC2_G3D_SHIFT (0) +#define S5P_CLKSRC2_G3D_MASK (0x3 << S5P_CLKSRC2_G3D_SHIFT) +#define S5P_CLKSRC2_MFC_SHIFT (4) +#define S5P_CLKSRC2_MFC_MASK (0x3 << S5P_CLKSRC2_MFC_SHIFT) + +/* CLKDIV0 */ +#define S5P_CLKDIV0_APLL_SHIFT (0) +#define S5P_CLKDIV0_APLL_MASK (0x7 << S5P_CLKDIV0_APLL_SHIFT) +#define S5P_CLKDIV0_A2M_SHIFT (4) +#define S5P_CLKDIV0_A2M_MASK (0x7 << S5P_CLKDIV0_A2M_SHIFT) +#define S5P_CLKDIV0_HCLK200_SHIFT (8) +#define S5P_CLKDIV0_HCLK200_MASK (0x7 << S5P_CLKDIV0_HCLK200_SHIFT) +#define S5P_CLKDIV0_PCLK100_SHIFT (12) +#define S5P_CLKDIV0_PCLK100_MASK (0x7 << S5P_CLKDIV0_PCLK100_SHIFT) +#define S5P_CLKDIV0_HCLK166_SHIFT (16) +#define S5P_CLKDIV0_HCLK166_MASK (0xF << S5P_CLKDIV0_HCLK166_SHIFT) +#define S5P_CLKDIV0_PCLK83_SHIFT (20) +#define S5P_CLKDIV0_PCLK83_MASK (0x7 << S5P_CLKDIV0_PCLK83_SHIFT) +#define S5P_CLKDIV0_HCLK133_SHIFT (24) +#define S5P_CLKDIV0_HCLK133_MASK (0xF << S5P_CLKDIV0_HCLK133_SHIFT) +#define S5P_CLKDIV0_PCLK66_SHIFT (28) +#define S5P_CLKDIV0_PCLK66_MASK (0x7 << S5P_CLKDIV0_PCLK66_SHIFT) + +/* CLKDIV2 */ +#define S5P_CLKDIV2_G3D_SHIFT (0) +#define S5P_CLKDIV2_G3D_MASK (0xF << S5P_CLKDIV2_G3D_SHIFT) +#define S5P_CLKDIV2_MFC_SHIFT (4) +#define S5P_CLKDIV2_MFC_MASK (0xF << S5P_CLKDIV2_MFC_SHIFT) + +/* CLKDIV6 */ +#define S5P_CLKDIV6_ONEDRAM_SHIFT (28) +#define S5P_CLKDIV6_ONEDRAM_MASK (0xF << S5P_CLKDIV6_ONEDRAM_SHIFT) -static struct clk *cpu_clk; static struct clk *dmc0_clk; static struct clk *dmc1_clk; -static struct cpufreq_freqs freqs; static DEFINE_MUTEX(set_freq_lock); /* APLL M,P,S values for 1G/800Mhz */ @@ -36,16 +91,7 @@ static DEFINE_MUTEX(set_freq_lock); /* Use 800MHz when entering sleep mode */ #define SLEEP_FREQ (800 * 1000) -/* - * relation has an additional symantics other than the standard of cpufreq - * DISALBE_FURTHER_CPUFREQ: disable further access to target - * ENABLE_FURTUER_CPUFREQ: enable access to target - */ -enum cpufreq_access { - DISABLE_FURTHER_CPUFREQ = 0x10, - ENABLE_FURTHER_CPUFREQ = 0x20, -}; - +/* Tracks if CPU frequency can be updated anymore */ static bool no_cpufreq_access; /* @@ -76,12 +122,12 @@ enum s5pv210_dmc_port { }; static struct cpufreq_frequency_table s5pv210_freq_table[] = { - {L0, 1000*1000}, - {L1, 800*1000}, - {L2, 400*1000}, - {L3, 200*1000}, - {L4, 100*1000}, - {0, CPUFREQ_TABLE_END}, + {0, L0, 1000*1000}, + {0, L1, 800*1000}, + {0, L2, 400*1000}, + {0, L3, 200*1000}, + {0, L4, 100*1000}, + {0, 0, CPUFREQ_TABLE_END}, }; static struct regulator *arm_regulator; @@ -144,7 +190,7 @@ static u32 clkdiv_val[5][11] = { /* * This function set DRAM refresh counter - * accoriding to operating frequency of DRAM + * according to operating frequency of DRAM * ch: DMC port number 0 or 1 * freq: Operating frequency of DRAM(KHz) */ @@ -154,96 +200,55 @@ static void s5pv210_set_refresh(enum s5pv210_dmc_port ch, unsigned long freq) void __iomem *reg = NULL; if (ch == DMC0) { - reg = (S5P_VA_DMC0 + 0x30); + reg = (dmc_base[0] + 0x30); } else if (ch == DMC1) { - reg = (S5P_VA_DMC1 + 0x30); + reg = (dmc_base[1] + 0x30); } else { - printk(KERN_ERR "Cannot find DMC port\n"); + pr_err("Cannot find DMC port\n"); return; } /* Find current DRAM frequency */ tmp = s5pv210_dram_conf[ch].freq; - do_div(tmp, freq); + tmp /= freq; tmp1 = s5pv210_dram_conf[ch].refresh; - do_div(tmp1, tmp); + tmp1 /= tmp; - __raw_writel(tmp1, reg); + writel_relaxed(tmp1, reg); } -static int s5pv210_verify_speed(struct cpufreq_policy *policy) -{ - if (policy->cpu) - return -EINVAL; - - return cpufreq_frequency_table_verify(policy, s5pv210_freq_table); -} - -static unsigned int s5pv210_getspeed(unsigned int cpu) -{ - if (cpu) - return 0; - - return clk_get_rate(cpu_clk) / 1000; -} - -static int s5pv210_target(struct cpufreq_policy *policy, - unsigned int target_freq, - unsigned int relation) +static int s5pv210_target(struct cpufreq_policy *policy, unsigned int index) { unsigned long reg; - unsigned int index, priv_index; + unsigned int priv_index; unsigned int pll_changing = 0; unsigned int bus_speed_changing = 0; + unsigned int old_freq, new_freq; int arm_volt, int_volt; int ret = 0; mutex_lock(&set_freq_lock); - if (relation & ENABLE_FURTHER_CPUFREQ) - no_cpufreq_access = false; - if (no_cpufreq_access) { -#ifdef CONFIG_PM_VERBOSE - pr_err("%s:%d denied access to %s as it is disabled" - "temporarily\n", __FILE__, __LINE__, __func__); -#endif + pr_err("Denied access to %s as it is disabled temporarily\n", + __func__); ret = -EINVAL; goto exit; } - if (relation & DISABLE_FURTHER_CPUFREQ) - no_cpufreq_access = true; - - relation &= ~(ENABLE_FURTHER_CPUFREQ | DISABLE_FURTHER_CPUFREQ); - - freqs.old = s5pv210_getspeed(0); - - if (cpufreq_frequency_table_target(policy, s5pv210_freq_table, - target_freq, relation, &index)) { - ret = -EINVAL; - goto exit; - } - - freqs.new = s5pv210_freq_table[index].frequency; - - if (freqs.new == freqs.old) - goto exit; + old_freq = policy->cur; + new_freq = s5pv210_freq_table[index].frequency; /* Finding current running level index */ - if (cpufreq_frequency_table_target(policy, s5pv210_freq_table, - freqs.old, relation, &priv_index)) { - ret = -EINVAL; - goto exit; - } + priv_index = cpufreq_table_find_index_h(policy, old_freq, false); arm_volt = dvs_conf[index].arm_volt; int_volt = dvs_conf[index].int_volt; - if (freqs.new > freqs.old) { + if (new_freq > old_freq) { ret = regulator_set_voltage(arm_regulator, arm_volt, arm_volt_max); if (ret) @@ -255,8 +260,6 @@ static int s5pv210_target(struct cpufreq_policy *policy, goto exit; } - cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE); - /* Check if there need to change PLL */ if ((index == L0) || (priv_index == L0)) pll_changing = 1; @@ -290,53 +293,53 @@ static int s5pv210_target(struct cpufreq_policy *policy, * 1. Temporary Change divider for MFC and G3D * SCLKA2M(200/1=200)->(200/4=50)Mhz */ - reg = __raw_readl(S5P_CLK_DIV2); + reg = readl_relaxed(S5P_CLK_DIV2); reg &= ~(S5P_CLKDIV2_G3D_MASK | S5P_CLKDIV2_MFC_MASK); reg |= (3 << S5P_CLKDIV2_G3D_SHIFT) | (3 << S5P_CLKDIV2_MFC_SHIFT); - __raw_writel(reg, S5P_CLK_DIV2); + writel_relaxed(reg, S5P_CLK_DIV2); /* For MFC, G3D dividing */ do { - reg = __raw_readl(S5P_CLKDIV_STAT0); + reg = readl_relaxed(S5P_CLKDIV_STAT0); } while (reg & ((1 << 16) | (1 << 17))); /* * 2. Change SCLKA2M(200Mhz)to SCLKMPLL in MFC_MUX, G3D MUX * (200/4=50)->(667/4=166)Mhz */ - reg = __raw_readl(S5P_CLK_SRC2); + reg = readl_relaxed(S5P_CLK_SRC2); reg &= ~(S5P_CLKSRC2_G3D_MASK | S5P_CLKSRC2_MFC_MASK); reg |= (1 << S5P_CLKSRC2_G3D_SHIFT) | (1 << S5P_CLKSRC2_MFC_SHIFT); - __raw_writel(reg, S5P_CLK_SRC2); + writel_relaxed(reg, S5P_CLK_SRC2); do { - reg = __raw_readl(S5P_CLKMUX_STAT1); + reg = readl_relaxed(S5P_CLKMUX_STAT1); } while (reg & ((1 << 7) | (1 << 3))); /* * 3. DMC1 refresh count for 133Mhz if (index == L4) is - * true refresh counter is already programed in upper + * true refresh counter is already programmed in upper * code. 0x287@83Mhz */ if (!bus_speed_changing) s5pv210_set_refresh(DMC1, 133000); /* 4. SCLKAPLL -> SCLKMPLL */ - reg = __raw_readl(S5P_CLK_SRC0); + reg = readl_relaxed(S5P_CLK_SRC0); reg &= ~(S5P_CLKSRC0_MUX200_MASK); reg |= (0x1 << S5P_CLKSRC0_MUX200_SHIFT); - __raw_writel(reg, S5P_CLK_SRC0); + writel_relaxed(reg, S5P_CLK_SRC0); do { - reg = __raw_readl(S5P_CLKMUX_STAT0); + reg = readl_relaxed(S5P_CLKMUX_STAT0); } while (reg & (0x1 << 18)); } /* Change divider */ - reg = __raw_readl(S5P_CLK_DIV0); + reg = readl_relaxed(S5P_CLK_DIV0); reg &= ~(S5P_CLKDIV0_APLL_MASK | S5P_CLKDIV0_A2M_MASK | S5P_CLKDIV0_HCLK200_MASK | S5P_CLKDIV0_PCLK100_MASK | @@ -352,78 +355,78 @@ static int s5pv210_target(struct cpufreq_policy *policy, (clkdiv_val[index][6] << S5P_CLKDIV0_HCLK133_SHIFT) | (clkdiv_val[index][7] << S5P_CLKDIV0_PCLK66_SHIFT)); - __raw_writel(reg, S5P_CLK_DIV0); + writel_relaxed(reg, S5P_CLK_DIV0); do { - reg = __raw_readl(S5P_CLKDIV_STAT0); + reg = readl_relaxed(S5P_CLKDIV_STAT0); } while (reg & 0xff); /* ARM MCS value changed */ - reg = __raw_readl(S5P_ARM_MCS_CON); + reg = readl_relaxed(S5P_ARM_MCS_CON); reg &= ~0x3; if (index >= L3) reg |= 0x3; else reg |= 0x1; - __raw_writel(reg, S5P_ARM_MCS_CON); + writel_relaxed(reg, S5P_ARM_MCS_CON); if (pll_changing) { /* 5. Set Lock time = 30us*24Mhz = 0x2cf */ - __raw_writel(0x2cf, S5P_APLL_LOCK); + writel_relaxed(0x2cf, S5P_APLL_LOCK); /* * 6. Turn on APLL * 6-1. Set PMS values - * 6-2. Wait untile the PLL is locked + * 6-2. Wait until the PLL is locked */ if (index == L0) - __raw_writel(APLL_VAL_1000, S5P_APLL_CON); + writel_relaxed(APLL_VAL_1000, S5P_APLL_CON); else - __raw_writel(APLL_VAL_800, S5P_APLL_CON); + writel_relaxed(APLL_VAL_800, S5P_APLL_CON); do { - reg = __raw_readl(S5P_APLL_CON); + reg = readl_relaxed(S5P_APLL_CON); } while (!(reg & (0x1 << 29))); /* - * 7. Change souce clock from SCLKMPLL(667Mhz) + * 7. Change source clock from SCLKMPLL(667Mhz) * to SCLKA2M(200Mhz) in MFC_MUX and G3D MUX * (667/4=166)->(200/4=50)Mhz */ - reg = __raw_readl(S5P_CLK_SRC2); + reg = readl_relaxed(S5P_CLK_SRC2); reg &= ~(S5P_CLKSRC2_G3D_MASK | S5P_CLKSRC2_MFC_MASK); reg |= (0 << S5P_CLKSRC2_G3D_SHIFT) | (0 << S5P_CLKSRC2_MFC_SHIFT); - __raw_writel(reg, S5P_CLK_SRC2); + writel_relaxed(reg, S5P_CLK_SRC2); do { - reg = __raw_readl(S5P_CLKMUX_STAT1); + reg = readl_relaxed(S5P_CLKMUX_STAT1); } while (reg & ((1 << 7) | (1 << 3))); /* * 8. Change divider for MFC and G3D * (200/4=50)->(200/1=200)Mhz */ - reg = __raw_readl(S5P_CLK_DIV2); + reg = readl_relaxed(S5P_CLK_DIV2); reg &= ~(S5P_CLKDIV2_G3D_MASK | S5P_CLKDIV2_MFC_MASK); reg |= (clkdiv_val[index][10] << S5P_CLKDIV2_G3D_SHIFT) | (clkdiv_val[index][9] << S5P_CLKDIV2_MFC_SHIFT); - __raw_writel(reg, S5P_CLK_DIV2); + writel_relaxed(reg, S5P_CLK_DIV2); /* For MFC, G3D dividing */ do { - reg = __raw_readl(S5P_CLKDIV_STAT0); + reg = readl_relaxed(S5P_CLKDIV_STAT0); } while (reg & ((1 << 16) | (1 << 17))); /* 9. Change MPLL to APLL in MSYS_MUX */ - reg = __raw_readl(S5P_CLK_SRC0); + reg = readl_relaxed(S5P_CLK_SRC0); reg &= ~(S5P_CLKSRC0_MUX200_MASK); reg |= (0x0 << S5P_CLKSRC0_MUX200_SHIFT); - __raw_writel(reg, S5P_CLK_SRC0); + writel_relaxed(reg, S5P_CLK_SRC0); do { - reg = __raw_readl(S5P_CLKMUX_STAT0); + reg = readl_relaxed(S5P_CLKMUX_STAT0); } while (reg & (0x1 << 18)); /* @@ -436,17 +439,17 @@ static int s5pv210_target(struct cpufreq_policy *policy, } /* - * L4 level need to change memory bus speed, hence onedram clock divier - * and memory refresh parameter should be changed + * L4 level needs to change memory bus speed, hence ONEDRAM clock + * divider and memory refresh parameter should be changed */ if (bus_speed_changing) { - reg = __raw_readl(S5P_CLK_DIV6); + reg = readl_relaxed(S5P_CLK_DIV6); reg &= ~S5P_CLKDIV6_ONEDRAM_MASK; reg |= (clkdiv_val[index][8] << S5P_CLKDIV6_ONEDRAM_SHIFT); - __raw_writel(reg, S5P_CLK_DIV6); + writel_relaxed(reg, S5P_CLK_DIV6); do { - reg = __raw_readl(S5P_CLKDIV_STAT1); + reg = readl_relaxed(S5P_CLKDIV_STAT1); } while (reg & (1 << 15)); /* Reconfigure DRAM refresh counter value */ @@ -467,9 +470,7 @@ static int s5pv210_target(struct cpufreq_policy *policy, } } - cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); - - if (freqs.new < freqs.old) { + if (new_freq < old_freq) { regulator_set_voltage(int_regulator, int_volt, int_volt_max); @@ -477,43 +478,31 @@ static int s5pv210_target(struct cpufreq_policy *policy, arm_volt, arm_volt_max); } - printk(KERN_DEBUG "Perf changed[L%d]\n", index); + pr_debug("Perf changed[L%d]\n", index); exit: mutex_unlock(&set_freq_lock); return ret; } -#ifdef CONFIG_PM -static int s5pv210_cpufreq_suspend(struct cpufreq_policy *policy) -{ - return 0; -} - -static int s5pv210_cpufreq_resume(struct cpufreq_policy *policy) -{ - return 0; -} -#endif - static int check_mem_type(void __iomem *dmc_reg) { unsigned long val; - val = __raw_readl(dmc_reg + 0x4); + val = readl_relaxed(dmc_reg + 0x4); val = (val & (0xf << 8)); return val >> 8; } -static int __init s5pv210_cpu_init(struct cpufreq_policy *policy) +static int s5pv210_cpu_init(struct cpufreq_policy *policy) { unsigned long mem_type; int ret; - cpu_clk = clk_get(NULL, "armclk"); - if (IS_ERR(cpu_clk)) - return PTR_ERR(cpu_clk); + policy->clk = clk_get(NULL, "armclk"); + if (IS_ERR(policy->clk)) + return PTR_ERR(policy->clk); dmc0_clk = clk_get(NULL, "sclk_dmc0"); if (IS_ERR(dmc0_clk)) { @@ -529,120 +518,170 @@ static int __init s5pv210_cpu_init(struct cpufreq_policy *policy) if (policy->cpu != 0) { ret = -EINVAL; - goto out_dmc1; + goto out; } /* * check_mem_type : This driver only support LPDDR & LPDDR2. * other memory type is not supported. */ - mem_type = check_mem_type(S5P_VA_DMC0); + mem_type = check_mem_type(dmc_base[0]); if ((mem_type != LPDDR) && (mem_type != LPDDR2)) { - printk(KERN_ERR "CPUFreq doesn't support this memory type\n"); + pr_err("CPUFreq doesn't support this memory type\n"); ret = -EINVAL; - goto out_dmc1; + goto out; } /* Find current refresh counter and frequency each DMC */ - s5pv210_dram_conf[0].refresh = (__raw_readl(S5P_VA_DMC0 + 0x30) * 1000); + s5pv210_dram_conf[0].refresh = (readl_relaxed(dmc_base[0] + 0x30) * 1000); s5pv210_dram_conf[0].freq = clk_get_rate(dmc0_clk); - s5pv210_dram_conf[1].refresh = (__raw_readl(S5P_VA_DMC1 + 0x30) * 1000); + s5pv210_dram_conf[1].refresh = (readl_relaxed(dmc_base[1] + 0x30) * 1000); s5pv210_dram_conf[1].freq = clk_get_rate(dmc1_clk); - policy->cur = policy->min = policy->max = s5pv210_getspeed(0); - - cpufreq_frequency_table_get_attr(s5pv210_freq_table, policy->cpu); - - policy->cpuinfo.transition_latency = 40000; - - return cpufreq_frequency_table_cpuinfo(policy, s5pv210_freq_table); + policy->suspend_freq = SLEEP_FREQ; + cpufreq_generic_init(policy, s5pv210_freq_table, 40000); + return 0; +out: + clk_put(dmc1_clk); out_dmc1: clk_put(dmc0_clk); out_dmc0: - clk_put(cpu_clk); + clk_put(policy->clk); return ret; } -static int s5pv210_cpufreq_notifier_event(struct notifier_block *this, - unsigned long event, void *ptr) +static int s5pv210_cpufreq_reboot_notifier_event(struct notifier_block *this, + unsigned long event, void *ptr) { + struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(0); int ret; - switch (event) { - case PM_SUSPEND_PREPARE: - ret = cpufreq_driver_target(cpufreq_cpu_get(0), SLEEP_FREQ, - DISABLE_FURTHER_CPUFREQ); - if (ret < 0) - return NOTIFY_BAD; - - return NOTIFY_OK; - case PM_POST_RESTORE: - case PM_POST_SUSPEND: - cpufreq_driver_target(cpufreq_cpu_get(0), SLEEP_FREQ, - ENABLE_FURTHER_CPUFREQ); - - return NOTIFY_OK; + if (!policy) { + pr_debug("cpufreq: get no policy for cpu0\n"); + return NOTIFY_BAD; } - return NOTIFY_DONE; -} - -static int s5pv210_cpufreq_reboot_notifier_event(struct notifier_block *this, - unsigned long event, void *ptr) -{ - int ret; + ret = cpufreq_driver_target(policy, SLEEP_FREQ, 0); - ret = cpufreq_driver_target(cpufreq_cpu_get(0), SLEEP_FREQ, - DISABLE_FURTHER_CPUFREQ); if (ret < 0) return NOTIFY_BAD; + no_cpufreq_access = true; return NOTIFY_DONE; } static struct cpufreq_driver s5pv210_driver = { - .flags = CPUFREQ_STICKY, - .verify = s5pv210_verify_speed, - .target = s5pv210_target, - .get = s5pv210_getspeed, + .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK, + .verify = cpufreq_generic_frequency_table_verify, + .target_index = s5pv210_target, + .get = cpufreq_generic_get, .init = s5pv210_cpu_init, .name = "s5pv210", -#ifdef CONFIG_PM - .suspend = s5pv210_cpufreq_suspend, - .resume = s5pv210_cpufreq_resume, -#endif -}; - -static struct notifier_block s5pv210_cpufreq_notifier = { - .notifier_call = s5pv210_cpufreq_notifier_event, + .suspend = cpufreq_generic_suspend, + .resume = cpufreq_generic_suspend, /* We need to set SLEEP FREQ again */ }; static struct notifier_block s5pv210_cpufreq_reboot_notifier = { .notifier_call = s5pv210_cpufreq_reboot_notifier_event, }; -static int __init s5pv210_cpufreq_init(void) +static int s5pv210_cpufreq_probe(struct platform_device *pdev) { + struct device *dev = &pdev->dev; + struct device_node *np; + int id, result = 0; + + /* + * HACK: This is a temporary workaround to get access to clock + * and DMC controller registers directly and remove static mappings + * and dependencies on platform headers. It is necessary to enable + * S5PV210 multi-platform support and will be removed together with + * this whole driver as soon as S5PV210 gets migrated to use + * cpufreq-dt driver. + */ arm_regulator = regulator_get(NULL, "vddarm"); - if (IS_ERR(arm_regulator)) { - pr_err("failed to get regulator vddarm"); - return PTR_ERR(arm_regulator); - } + if (IS_ERR(arm_regulator)) + return dev_err_probe(dev, PTR_ERR(arm_regulator), + "failed to get regulator vddarm\n"); int_regulator = regulator_get(NULL, "vddint"); if (IS_ERR(int_regulator)) { - pr_err("failed to get regulator vddint"); - regulator_put(arm_regulator); - return PTR_ERR(int_regulator); + result = dev_err_probe(dev, PTR_ERR(int_regulator), + "failed to get regulator vddint\n"); + goto err_int_regulator; + } + + np = of_find_compatible_node(NULL, NULL, "samsung,s5pv210-clock"); + if (!np) { + dev_err(dev, "failed to find clock controller DT node\n"); + result = -ENODEV; + goto err_clock; + } + + clk_base = of_iomap(np, 0); + of_node_put(np); + if (!clk_base) { + dev_err(dev, "failed to map clock registers\n"); + result = -EFAULT; + goto err_clock; + } + + for_each_compatible_node(np, NULL, "samsung,s5pv210-dmc") { + id = of_alias_get_id(np, "dmc"); + if (id < 0 || id >= ARRAY_SIZE(dmc_base)) { + dev_err(dev, "failed to get alias of dmc node '%pOFn'\n", np); + of_node_put(np); + result = id; + goto err_clk_base; + } + + dmc_base[id] = of_iomap(np, 0); + if (!dmc_base[id]) { + dev_err(dev, "failed to map dmc%d registers\n", id); + of_node_put(np); + result = -EFAULT; + goto err_dmc; + } + } + + for (id = 0; id < ARRAY_SIZE(dmc_base); ++id) { + if (!dmc_base[id]) { + dev_err(dev, "failed to find dmc%d node\n", id); + result = -ENODEV; + goto err_dmc; + } } - register_pm_notifier(&s5pv210_cpufreq_notifier); register_reboot_notifier(&s5pv210_cpufreq_reboot_notifier); return cpufreq_register_driver(&s5pv210_driver); + +err_dmc: + for (id = 0; id < ARRAY_SIZE(dmc_base); ++id) + if (dmc_base[id]) { + iounmap(dmc_base[id]); + dmc_base[id] = NULL; + } + +err_clk_base: + iounmap(clk_base); + +err_clock: + regulator_put(int_regulator); + +err_int_regulator: + regulator_put(arm_regulator); + + return result; } -late_initcall(s5pv210_cpufreq_init); +static struct platform_driver s5pv210_cpufreq_platdrv = { + .driver = { + .name = "s5pv210-cpufreq", + }, + .probe = s5pv210_cpufreq_probe, +}; +builtin_platform_driver(s5pv210_cpufreq_platdrv); diff --git a/drivers/cpufreq/sa1100-cpufreq.c b/drivers/cpufreq/sa1100-cpufreq.c deleted file mode 100644 index cff18e87ca58..000000000000 --- a/drivers/cpufreq/sa1100-cpufreq.c +++ /dev/null @@ -1,247 +0,0 @@ -/* - * cpu-sa1100.c: clock scaling for the SA1100 - * - * Copyright (C) 2000 2001, The Delft University of Technology - * - * Authors: - * - Johan Pouwelse (J.A.Pouwelse@its.tudelft.nl): initial version - * - Erik Mouw (J.A.K.Mouw@its.tudelft.nl): - * - major rewrite for linux-2.3.99 - * - rewritten for the more generic power management scheme in - * linux-2.4.5-rmk1 - * - * This software has been developed while working on the LART - * computing board (http://www.lartmaker.nl/), which is - * sponsored by the Mobile Multi-media Communications - * (http://www.mobimedia.org/) and Ubiquitous Communications - * (http://www.ubicom.tudelft.nl/) projects. - * - * The authors can be reached at: - * - * Erik Mouw - * Information and Communication Theory Group - * Faculty of Information Technology and Systems - * Delft University of Technology - * P.O. Box 5031 - * 2600 GA Delft - * The Netherlands - * - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - * - * - * Theory of operations - * ==================== - * - * Clock scaling can be used to lower the power consumption of the CPU - * core. This will give you a somewhat longer running time. - * - * The SA-1100 has a single register to change the core clock speed: - * - * PPCR 0x90020014 PLL config - * - * However, the DRAM timings are closely related to the core clock - * speed, so we need to change these, too. The used registers are: - * - * MDCNFG 0xA0000000 DRAM config - * MDCAS0 0xA0000004 Access waveform - * MDCAS1 0xA0000008 Access waveform - * MDCAS2 0xA000000C Access waveform - * - * Care must be taken to change the DRAM parameters the correct way, - * because otherwise the DRAM becomes unusable and the kernel will - * crash. - * - * The simple solution to avoid a kernel crash is to put the actual - * clock change in ROM and jump to that code from the kernel. The main - * disadvantage is that the ROM has to be modified, which is not - * possible on all SA-1100 platforms. Another disadvantage is that - * jumping to ROM makes clock switching unnecessary complicated. - * - * The idea behind this driver is that the memory configuration can be - * changed while running from DRAM (even with interrupts turned on!) - * as long as all re-configuration steps yield a valid DRAM - * configuration. The advantages are clear: it will run on all SA-1100 - * platforms, and the code is very simple. - * - * If you really want to understand what is going on in - * sa1100_update_dram_timings(), you'll have to read sections 8.2, - * 9.5.7.3, and 10.2 from the "Intel StrongARM SA-1100 Microprocessor - * Developers Manual" (available for free from Intel). - * - */ - -#include <linux/kernel.h> -#include <linux/types.h> -#include <linux/init.h> -#include <linux/cpufreq.h> -#include <linux/io.h> - -#include <asm/cputype.h> - -#include <mach/generic.h> -#include <mach/hardware.h> - -struct sa1100_dram_regs { - int speed; - u32 mdcnfg; - u32 mdcas0; - u32 mdcas1; - u32 mdcas2; -}; - - -static struct cpufreq_driver sa1100_driver; - -static struct sa1100_dram_regs sa1100_dram_settings[] = { - /*speed, mdcnfg, mdcas0, mdcas1, mdcas2, clock freq */ - { 59000, 0x00dc88a3, 0xcccccccf, 0xfffffffc, 0xffffffff},/* 59.0 MHz */ - { 73700, 0x011490a3, 0xcccccccf, 0xfffffffc, 0xffffffff},/* 73.7 MHz */ - { 88500, 0x014e90a3, 0xcccccccf, 0xfffffffc, 0xffffffff},/* 88.5 MHz */ - {103200, 0x01889923, 0xcccccccf, 0xfffffffc, 0xffffffff},/* 103.2 MHz */ - {118000, 0x01c29923, 0x9999998f, 0xfffffff9, 0xffffffff},/* 118.0 MHz */ - {132700, 0x01fb2123, 0x9999998f, 0xfffffff9, 0xffffffff},/* 132.7 MHz */ - {147500, 0x02352123, 0x3333330f, 0xfffffff3, 0xffffffff},/* 147.5 MHz */ - {162200, 0x026b29a3, 0x38e38e1f, 0xfff8e38e, 0xffffffff},/* 162.2 MHz */ - {176900, 0x02a329a3, 0x71c71c1f, 0xfff1c71c, 0xffffffff},/* 176.9 MHz */ - {191700, 0x02dd31a3, 0xe38e383f, 0xffe38e38, 0xffffffff},/* 191.7 MHz */ - {206400, 0x03153223, 0xc71c703f, 0xffc71c71, 0xffffffff},/* 206.4 MHz */ - {221200, 0x034fba23, 0xc71c703f, 0xffc71c71, 0xffffffff},/* 221.2 MHz */ - {235900, 0x03853a23, 0xe1e1e07f, 0xe1e1e1e1, 0xffffffe1},/* 235.9 MHz */ - {250700, 0x03bf3aa3, 0xc3c3c07f, 0xc3c3c3c3, 0xffffffc3},/* 250.7 MHz */ - {265400, 0x03f7c2a3, 0xc3c3c07f, 0xc3c3c3c3, 0xffffffc3},/* 265.4 MHz */ - {280200, 0x0431c2a3, 0x878780ff, 0x87878787, 0xffffff87},/* 280.2 MHz */ - { 0, 0, 0, 0, 0 } /* last entry */ -}; - -static void sa1100_update_dram_timings(int current_speed, int new_speed) -{ - struct sa1100_dram_regs *settings = sa1100_dram_settings; - - /* find speed */ - while (settings->speed != 0) { - if (new_speed == settings->speed) - break; - - settings++; - } - - if (settings->speed == 0) { - panic("%s: couldn't find dram setting for speed %d\n", - __func__, new_speed); - } - - /* No risk, no fun: run with interrupts on! */ - if (new_speed > current_speed) { - /* We're going FASTER, so first relax the memory - * timings before changing the core frequency - */ - - /* Half the memory access clock */ - MDCNFG |= MDCNFG_CDB2; - - /* The order of these statements IS important, keep 8 - * pulses!! - */ - MDCAS2 = settings->mdcas2; - MDCAS1 = settings->mdcas1; - MDCAS0 = settings->mdcas0; - MDCNFG = settings->mdcnfg; - } else { - /* We're going SLOWER: first decrease the core - * frequency and then tighten the memory settings. - */ - - /* Half the memory access clock */ - MDCNFG |= MDCNFG_CDB2; - - /* The order of these statements IS important, keep 8 - * pulses!! - */ - MDCAS0 = settings->mdcas0; - MDCAS1 = settings->mdcas1; - MDCAS2 = settings->mdcas2; - MDCNFG = settings->mdcnfg; - } -} - -static int sa1100_target(struct cpufreq_policy *policy, - unsigned int target_freq, - unsigned int relation) -{ - unsigned int cur = sa11x0_getspeed(0); - unsigned int new_ppcr; - struct cpufreq_freqs freqs; - - new_ppcr = sa11x0_freq_to_ppcr(target_freq); - switch (relation) { - case CPUFREQ_RELATION_L: - if (sa11x0_ppcr_to_freq(new_ppcr) > policy->max) - new_ppcr--; - break; - case CPUFREQ_RELATION_H: - if ((sa11x0_ppcr_to_freq(new_ppcr) > target_freq) && - (sa11x0_ppcr_to_freq(new_ppcr - 1) >= policy->min)) - new_ppcr--; - break; - } - - freqs.old = cur; - freqs.new = sa11x0_ppcr_to_freq(new_ppcr); - - cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE); - - if (freqs.new > cur) - sa1100_update_dram_timings(cur, freqs.new); - - PPCR = new_ppcr; - - if (freqs.new < cur) - sa1100_update_dram_timings(cur, freqs.new); - - cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); - - return 0; -} - -static int __init sa1100_cpu_init(struct cpufreq_policy *policy) -{ - if (policy->cpu != 0) - return -EINVAL; - policy->cur = policy->min = policy->max = sa11x0_getspeed(0); - policy->cpuinfo.min_freq = 59000; - policy->cpuinfo.max_freq = 287000; - policy->cpuinfo.transition_latency = CPUFREQ_ETERNAL; - return 0; -} - -static struct cpufreq_driver sa1100_driver __refdata = { - .flags = CPUFREQ_STICKY, - .verify = sa11x0_verify_speed, - .target = sa1100_target, - .get = sa11x0_getspeed, - .init = sa1100_cpu_init, - .name = "sa1100", -}; - -static int __init sa1100_dram_init(void) -{ - if (cpu_is_sa1100()) - return cpufreq_register_driver(&sa1100_driver); - else - return -ENODEV; -} - -arch_initcall(sa1100_dram_init); diff --git a/drivers/cpufreq/sa1110-cpufreq.c b/drivers/cpufreq/sa1110-cpufreq.c index 39c90b6f4286..bb7f591a8b05 100644 --- a/drivers/cpufreq/sa1110-cpufreq.c +++ b/drivers/cpufreq/sa1110-cpufreq.c @@ -1,12 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * linux/arch/arm/mach-sa1100/cpu-sa1110.c * * Copyright (C) 2001 Russell King * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * * Note: there are two erratas that apply to the SA1110 here: * 7 - SDRAM auto-power-up failure (rev A0) * 13 - Corruption of internal register reads/writes following @@ -159,7 +156,7 @@ sdram_calculate_timing(struct sdram_info *sd, u_int cpu_khz, * half speed or use delayed read latching (errata 13). */ if ((ns_to_cycles(sdram->tck, sd_khz) > 1) || - (CPU_REVISION < CPU_SA1110_B2 && sd_khz < 62000)) + (read_cpuid_revision() < ARM_CPU_REV_SA1110_B2 && sd_khz < 62000)) sd_khz /= 2; sd->mdcnfg = MDCNFG & 0x007f007f; @@ -229,36 +226,14 @@ sdram_update_refresh(u_int cpu_khz, struct sdram_params *sdram) /* * Ok, set the CPU frequency. */ -static int sa1110_target(struct cpufreq_policy *policy, - unsigned int target_freq, - unsigned int relation) +static int sa1110_target(struct cpufreq_policy *policy, unsigned int ppcr) { struct sdram_params *sdram = &sdram_params; - struct cpufreq_freqs freqs; struct sdram_info sd; unsigned long flags; - unsigned int ppcr, unused; - - switch (relation) { - case CPUFREQ_RELATION_L: - ppcr = sa11x0_freq_to_ppcr(target_freq); - if (sa11x0_ppcr_to_freq(ppcr) > policy->max) - ppcr--; - break; - case CPUFREQ_RELATION_H: - ppcr = sa11x0_freq_to_ppcr(target_freq); - if (ppcr && (sa11x0_ppcr_to_freq(ppcr) > target_freq) && - (sa11x0_ppcr_to_freq(ppcr-1) >= policy->min)) - ppcr--; - break; - default: - return -EINVAL; - } - - freqs.old = sa11x0_getspeed(0); - freqs.new = sa11x0_ppcr_to_freq(ppcr); + unsigned int unused; - sdram_calculate_timing(&sd, freqs.new, sdram); + sdram_calculate_timing(&sd, sa11x0_freq_table[ppcr].frequency, sdram); #if 0 /* @@ -277,8 +252,6 @@ static int sa1110_target(struct cpufreq_policy *policy, sd.mdcas[2] = 0xaaaaaaaa; #endif - cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE); - /* * The clock could be going away for some time. Set the SDRAMs * to refresh rapidly (every 64 memory clock cycles). To get @@ -323,30 +296,24 @@ static int sa1110_target(struct cpufreq_policy *policy, /* * Now, return the SDRAM refresh back to normal. */ - sdram_update_refresh(freqs.new, sdram); - - cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); + sdram_update_refresh(sa11x0_freq_table[ppcr].frequency, sdram); return 0; } static int __init sa1110_cpu_init(struct cpufreq_policy *policy) { - if (policy->cpu != 0) - return -EINVAL; - policy->cur = policy->min = policy->max = sa11x0_getspeed(0); - policy->cpuinfo.min_freq = 59000; - policy->cpuinfo.max_freq = 287000; - policy->cpuinfo.transition_latency = CPUFREQ_ETERNAL; + cpufreq_generic_init(policy, sa11x0_freq_table, 0); return 0; } /* sa1110_driver needs __refdata because it must remain after init registers * it with cpufreq_register_driver() */ static struct cpufreq_driver sa1110_driver __refdata = { - .flags = CPUFREQ_STICKY, - .verify = sa11x0_verify_speed, - .target = sa1110_target, + .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK | + CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING, + .verify = cpufreq_generic_frequency_table_verify, + .target_index = sa1110_target, .get = sa11x0_getspeed, .init = sa1110_cpu_init, .name = "sa1110", @@ -377,14 +344,8 @@ static int __init sa1110_clk_init(void) if (!name[0]) { if (machine_is_assabet()) name = "TC59SM716-CL3"; - if (machine_is_pt_system3()) - name = "K4S641632D"; - if (machine_is_h3100()) - name = "KM416S4030CT"; - if (machine_is_jornada720()) + if (machine_is_jornada720() || machine_is_h3600()) name = "K4S281632B-1H"; - if (machine_is_nanoengine()) - name = "MT48LC8M16A2TG-75"; } sdram = sa1110_find_sdram(name); diff --git a/drivers/cpufreq/sc520_freq.c b/drivers/cpufreq/sc520_freq.c index 77a210975fc4..b360f03a116f 100644 --- a/drivers/cpufreq/sc520_freq.c +++ b/drivers/cpufreq/sc520_freq.c @@ -1,18 +1,16 @@ +// SPDX-License-Identifier: GPL-2.0-or-later /* * sc520_freq.c: cpufreq driver for the AMD Elan sc520 * * Copyright (C) 2005 Sean Young <sean@mess.org> * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - * * Based on elanfreq.c * * 2005-03-30: - initial revision */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + #include <linux/kernel.h> #include <linux/module.h> #include <linux/init.h> @@ -23,19 +21,16 @@ #include <linux/io.h> #include <asm/cpu_device_id.h> -#include <asm/msr.h> #define MMCR_BASE 0xfffef000 /* The default base address */ #define OFFS_CPUCTL 0x2 /* CPU Control Register */ static __u8 __iomem *cpuctl; -#define PFX "sc520_freq: " - static struct cpufreq_frequency_table sc520_freq_table[] = { - {0x01, 100000}, - {0x02, 133000}, - {0, CPUFREQ_TABLE_END}, + {0, 0x01, 100000}, + {0, 0x02, 133000}, + {0, 0, CPUFREQ_TABLE_END}, }; static unsigned int sc520_freq_get_cpu_frequency(unsigned int cpu) @@ -44,8 +39,9 @@ static unsigned int sc520_freq_get_cpu_frequency(unsigned int cpu) switch (clockspeed_reg & 0x03) { default: - printk(KERN_ERR PFX "error: cpuctl register has unexpected " - "value %02x\n", clockspeed_reg); + pr_err("error: cpuctl register has unexpected value %02x\n", + clockspeed_reg); + fallthrough; case 0x01: return 100000; case 0x02: @@ -53,21 +49,11 @@ static unsigned int sc520_freq_get_cpu_frequency(unsigned int cpu) } } -static void sc520_freq_set_cpu_state(struct cpufreq_policy *policy, - unsigned int state) +static int sc520_freq_target(struct cpufreq_policy *policy, unsigned int state) { - struct cpufreq_freqs freqs; u8 clockspeed_reg; - freqs.old = sc520_freq_get_cpu_frequency(0); - freqs.new = sc520_freq_table[state].frequency; - - cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE); - - pr_debug("attempting to set frequency to %i kHz\n", - sc520_freq_table[state].frequency); - local_irq_disable(); clockspeed_reg = *cpuctl & ~0x03; @@ -75,30 +61,9 @@ static void sc520_freq_set_cpu_state(struct cpufreq_policy *policy, local_irq_enable(); - cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); -}; - -static int sc520_freq_verify(struct cpufreq_policy *policy) -{ - return cpufreq_frequency_table_verify(policy, &sc520_freq_table[0]); -} - -static int sc520_freq_target(struct cpufreq_policy *policy, - unsigned int target_freq, - unsigned int relation) -{ - unsigned int newstate = 0; - - if (cpufreq_frequency_table_target(policy, sc520_freq_table, - target_freq, relation, &newstate)) - return -EINVAL; - - sc520_freq_set_cpu_state(policy, newstate); - return 0; } - /* * Module init and exit code */ @@ -106,7 +71,6 @@ static int sc520_freq_target(struct cpufreq_policy *policy, static int sc520_freq_cpu_init(struct cpufreq_policy *policy) { struct cpuinfo_x86 *c = &cpu_data(0); - int result; /* capability check */ if (c->x86_vendor != X86_VENDOR_AMD || @@ -115,44 +79,22 @@ static int sc520_freq_cpu_init(struct cpufreq_policy *policy) /* cpuinfo and default policy values */ policy->cpuinfo.transition_latency = 1000000; /* 1ms */ - policy->cur = sc520_freq_get_cpu_frequency(0); - - result = cpufreq_frequency_table_cpuinfo(policy, sc520_freq_table); - if (result) - return result; - - cpufreq_frequency_table_get_attr(sc520_freq_table, policy->cpu); + policy->freq_table = sc520_freq_table; return 0; } -static int sc520_freq_cpu_exit(struct cpufreq_policy *policy) -{ - cpufreq_frequency_table_put_attr(policy->cpu); - return 0; -} - - -static struct freq_attr *sc520_freq_attr[] = { - &cpufreq_freq_attr_scaling_available_freqs, - NULL, -}; - - static struct cpufreq_driver sc520_freq_driver = { .get = sc520_freq_get_cpu_frequency, - .verify = sc520_freq_verify, - .target = sc520_freq_target, + .verify = cpufreq_generic_frequency_table_verify, + .target_index = sc520_freq_target, .init = sc520_freq_cpu_init, - .exit = sc520_freq_cpu_exit, .name = "sc520_freq", - .owner = THIS_MODULE, - .attr = sc520_freq_attr, }; static const struct x86_cpu_id sc520_ids[] = { - { X86_VENDOR_AMD, 4, 9 }, + X86_MATCH_VENDOR_FAM_MODEL(AMD, 4, 9, NULL), {} }; MODULE_DEVICE_TABLE(x86cpu, sc520_ids); @@ -166,7 +108,7 @@ static int __init sc520_freq_init(void) cpuctl = ioremap((unsigned long)(MMCR_BASE + OFFS_CPUCTL), 1); if (!cpuctl) { - printk(KERN_ERR "sc520_freq: error: failed to remap memory\n"); + pr_err("sc520_freq: error: failed to remap memory\n"); return -ENOMEM; } diff --git a/drivers/cpufreq/scmi-cpufreq.c b/drivers/cpufreq/scmi-cpufreq.c new file mode 100644 index 000000000000..d2a110079f5f --- /dev/null +++ b/drivers/cpufreq/scmi-cpufreq.c @@ -0,0 +1,496 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * System Control and Power Interface (SCMI) based CPUFreq Interface driver + * + * Copyright (C) 2018-2021 ARM Ltd. + * Sudeep Holla <sudeep.holla@arm.com> + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/clk-provider.h> +#include <linux/cpu.h> +#include <linux/cpufreq.h> +#include <linux/cpumask.h> +#include <linux/energy_model.h> +#include <linux/export.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/pm_opp.h> +#include <linux/pm_qos.h> +#include <linux/slab.h> +#include <linux/scmi_protocol.h> +#include <linux/types.h> +#include <linux/units.h> + +struct scmi_data { + int domain_id; + int nr_opp; + struct device *cpu_dev; + cpumask_var_t opp_shared_cpus; + struct notifier_block limit_notify_nb; + struct freq_qos_request limits_freq_req; +}; + +static struct scmi_protocol_handle *ph; +static const struct scmi_perf_proto_ops *perf_ops; +static struct cpufreq_driver scmi_cpufreq_driver; + +static unsigned int scmi_cpufreq_get_rate(unsigned int cpu) +{ + struct cpufreq_policy *policy; + struct scmi_data *priv; + unsigned long rate; + int ret; + + policy = cpufreq_cpu_get_raw(cpu); + if (unlikely(!policy)) + return 0; + + priv = policy->driver_data; + + ret = perf_ops->freq_get(ph, priv->domain_id, &rate, false); + if (ret) + return 0; + return rate / 1000; +} + +/* + * perf_ops->freq_set is not a synchronous, the actual OPP change will + * happen asynchronously and can get notified if the events are + * subscribed for by the SCMI firmware + */ +static int +scmi_cpufreq_set_target(struct cpufreq_policy *policy, unsigned int index) +{ + struct scmi_data *priv = policy->driver_data; + u64 freq = policy->freq_table[index].frequency; + + return perf_ops->freq_set(ph, priv->domain_id, freq * 1000, false); +} + +static unsigned int scmi_cpufreq_fast_switch(struct cpufreq_policy *policy, + unsigned int target_freq) +{ + struct scmi_data *priv = policy->driver_data; + unsigned long freq = target_freq; + + if (!perf_ops->freq_set(ph, priv->domain_id, freq * 1000, true)) + return target_freq; + + return 0; +} + +static int scmi_cpu_domain_id(struct device *cpu_dev) +{ + struct device_node *np = cpu_dev->of_node; + struct of_phandle_args domain_id; + int index; + + if (of_parse_phandle_with_args(np, "clocks", "#clock-cells", 0, + &domain_id)) { + /* Find the corresponding index for power-domain "perf". */ + index = of_property_match_string(np, "power-domain-names", + "perf"); + if (index < 0) + return -EINVAL; + + if (of_parse_phandle_with_args(np, "power-domains", + "#power-domain-cells", index, + &domain_id)) + return -EINVAL; + } + + return domain_id.args[0]; +} + +static int +scmi_get_sharing_cpus(struct device *cpu_dev, int domain, + struct cpumask *cpumask) +{ + int cpu, tdomain; + struct device *tcpu_dev; + + for_each_present_cpu(cpu) { + if (cpu == cpu_dev->id) + continue; + + tcpu_dev = get_cpu_device(cpu); + if (!tcpu_dev) + continue; + + tdomain = scmi_cpu_domain_id(tcpu_dev); + if (tdomain == domain) + cpumask_set_cpu(cpu, cpumask); + } + + return 0; +} + +static int __maybe_unused +scmi_get_cpu_power(struct device *cpu_dev, unsigned long *power, + unsigned long *KHz) +{ + enum scmi_power_scale power_scale = perf_ops->power_scale_get(ph); + unsigned long Hz; + int ret, domain; + + domain = scmi_cpu_domain_id(cpu_dev); + if (domain < 0) + return domain; + + /* Get the power cost of the performance domain. */ + Hz = *KHz * 1000; + ret = perf_ops->est_power_get(ph, domain, &Hz, power); + if (ret) + return ret; + + /* Convert the power to uW if it is mW (ignore bogoW) */ + if (power_scale == SCMI_POWER_MILLIWATTS) + *power *= MICROWATT_PER_MILLIWATT; + + /* The EM framework specifies the frequency in KHz. */ + *KHz = Hz / 1000; + + return 0; +} + +static int +scmi_get_rate_limit(u32 domain, bool has_fast_switch) +{ + int ret, rate_limit; + + if (has_fast_switch) { + /* + * Fast channels are used whenever available, + * so use their rate_limit value if populated. + */ + ret = perf_ops->fast_switch_rate_limit(ph, domain, + &rate_limit); + if (!ret && rate_limit) + return rate_limit; + } + + ret = perf_ops->rate_limit_get(ph, domain, &rate_limit); + if (ret) + return 0; + + return rate_limit; +} + +static int scmi_limit_notify_cb(struct notifier_block *nb, unsigned long event, void *data) +{ + struct scmi_data *priv = container_of(nb, struct scmi_data, limit_notify_nb); + struct scmi_perf_limits_report *limit_notify = data; + unsigned int limit_freq_khz; + int ret; + + limit_freq_khz = limit_notify->range_max_freq / HZ_PER_KHZ; + + ret = freq_qos_update_request(&priv->limits_freq_req, limit_freq_khz); + if (ret < 0) + pr_warn("failed to update freq constraint: %d\n", ret); + + return NOTIFY_OK; +} + +static int scmi_cpufreq_init(struct cpufreq_policy *policy) +{ + int ret, nr_opp, domain; + unsigned int latency; + struct device *cpu_dev; + struct scmi_data *priv; + struct cpufreq_frequency_table *freq_table; + struct scmi_device *sdev = cpufreq_get_driver_data(); + + cpu_dev = get_cpu_device(policy->cpu); + if (!cpu_dev) { + pr_err("failed to get cpu%d device\n", policy->cpu); + return -ENODEV; + } + + domain = scmi_cpu_domain_id(cpu_dev); + if (domain < 0) + return domain; + + priv = kzalloc(sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + if (!zalloc_cpumask_var(&priv->opp_shared_cpus, GFP_KERNEL)) { + ret = -ENOMEM; + goto out_free_priv; + } + + /* Obtain CPUs that share SCMI performance controls */ + ret = scmi_get_sharing_cpus(cpu_dev, domain, policy->cpus); + if (ret) { + dev_warn(cpu_dev, "failed to get sharing cpumask\n"); + goto out_free_cpumask; + } + + /* + * Obtain CPUs that share performance levels. + * The OPP 'sharing cpus' info may come from DT through an empty opp + * table and opp-shared. + */ + ret = dev_pm_opp_of_get_sharing_cpus(cpu_dev, priv->opp_shared_cpus); + if (ret || cpumask_empty(priv->opp_shared_cpus)) { + /* + * Either opp-table is not set or no opp-shared was found. + * Use the CPU mask from SCMI to designate CPUs sharing an OPP + * table. + */ + cpumask_copy(priv->opp_shared_cpus, policy->cpus); + } + + /* + * A previous CPU may have marked OPPs as shared for a few CPUs, based on + * what OPP core provided. If the current CPU is part of those few, then + * there is no need to add OPPs again. + */ + nr_opp = dev_pm_opp_get_opp_count(cpu_dev); + if (nr_opp <= 0) { + ret = perf_ops->device_opps_add(ph, cpu_dev, domain); + if (ret) { + dev_warn(cpu_dev, "failed to add opps to the device\n"); + goto out_free_cpumask; + } + + nr_opp = dev_pm_opp_get_opp_count(cpu_dev); + if (nr_opp <= 0) { + dev_err(cpu_dev, "%s: No OPPs for this device: %d\n", + __func__, nr_opp); + + ret = -ENODEV; + goto out_free_opp; + } + + ret = dev_pm_opp_set_sharing_cpus(cpu_dev, priv->opp_shared_cpus); + if (ret) { + dev_err(cpu_dev, "%s: failed to mark OPPs as shared: %d\n", + __func__, ret); + + goto out_free_opp; + } + + priv->nr_opp = nr_opp; + } + + ret = dev_pm_opp_init_cpufreq_table(cpu_dev, &freq_table); + if (ret) { + dev_err(cpu_dev, "failed to init cpufreq table: %d\n", ret); + goto out_free_opp; + } + + priv->cpu_dev = cpu_dev; + priv->domain_id = domain; + + policy->driver_data = priv; + policy->freq_table = freq_table; + + /* SCMI allows DVFS request for any domain from any CPU */ + policy->dvfs_possible_from_any_cpu = true; + + latency = perf_ops->transition_latency_get(ph, domain); + if (!latency) + latency = CPUFREQ_DEFAULT_TRANSITION_LATENCY_NS; + + policy->cpuinfo.transition_latency = latency; + + policy->fast_switch_possible = + perf_ops->fast_switch_possible(ph, domain); + + policy->transition_delay_us = + scmi_get_rate_limit(domain, policy->fast_switch_possible); + + ret = freq_qos_add_request(&policy->constraints, &priv->limits_freq_req, FREQ_QOS_MAX, + FREQ_QOS_MAX_DEFAULT_VALUE); + if (ret < 0) { + dev_err(cpu_dev, "failed to add qos limits request: %d\n", ret); + goto out_free_table; + } + + priv->limit_notify_nb.notifier_call = scmi_limit_notify_cb; + ret = sdev->handle->notify_ops->event_notifier_register(sdev->handle, SCMI_PROTOCOL_PERF, + SCMI_EVENT_PERFORMANCE_LIMITS_CHANGED, + &priv->domain_id, + &priv->limit_notify_nb); + if (ret) + dev_warn(&sdev->dev, + "failed to register for limits change notifier for domain %d\n", + priv->domain_id); + + return 0; + +out_free_table: + dev_pm_opp_free_cpufreq_table(cpu_dev, &freq_table); +out_free_opp: + dev_pm_opp_remove_all_dynamic(cpu_dev); + +out_free_cpumask: + free_cpumask_var(priv->opp_shared_cpus); + +out_free_priv: + kfree(priv); + + return ret; +} + +static void scmi_cpufreq_exit(struct cpufreq_policy *policy) +{ + struct scmi_data *priv = policy->driver_data; + struct scmi_device *sdev = cpufreq_get_driver_data(); + + sdev->handle->notify_ops->event_notifier_unregister(sdev->handle, SCMI_PROTOCOL_PERF, + SCMI_EVENT_PERFORMANCE_LIMITS_CHANGED, + &priv->domain_id, + &priv->limit_notify_nb); + freq_qos_remove_request(&priv->limits_freq_req); + dev_pm_opp_free_cpufreq_table(priv->cpu_dev, &policy->freq_table); + dev_pm_opp_remove_all_dynamic(priv->cpu_dev); + free_cpumask_var(priv->opp_shared_cpus); + kfree(priv); +} + +static void scmi_cpufreq_register_em(struct cpufreq_policy *policy) +{ + struct em_data_callback em_cb = EM_DATA_CB(scmi_get_cpu_power); + enum scmi_power_scale power_scale = perf_ops->power_scale_get(ph); + struct scmi_data *priv = policy->driver_data; + bool em_power_scale = false; + + /* + * This callback will be called for each policy, but we don't need to + * register with EM every time. Despite not being part of the same + * policy, some CPUs may still share their perf-domains, and a CPU from + * another policy may already have registered with EM on behalf of CPUs + * of this policy. + */ + if (!priv->nr_opp) + return; + + if (power_scale == SCMI_POWER_MILLIWATTS + || power_scale == SCMI_POWER_MICROWATTS) + em_power_scale = true; + + em_dev_register_perf_domain(get_cpu_device(policy->cpu), priv->nr_opp, + &em_cb, priv->opp_shared_cpus, + em_power_scale); +} + +static struct cpufreq_driver scmi_cpufreq_driver = { + .name = "scmi", + .flags = CPUFREQ_HAVE_GOVERNOR_PER_POLICY | + CPUFREQ_NEED_INITIAL_FREQ_CHECK | + CPUFREQ_IS_COOLING_DEV, + .verify = cpufreq_generic_frequency_table_verify, + .target_index = scmi_cpufreq_set_target, + .fast_switch = scmi_cpufreq_fast_switch, + .get = scmi_cpufreq_get_rate, + .init = scmi_cpufreq_init, + .exit = scmi_cpufreq_exit, + .register_em = scmi_cpufreq_register_em, + .set_boost = cpufreq_boost_set_sw, +}; + +static bool scmi_dev_used_by_cpus(struct device *scmi_dev) +{ + struct device_node *scmi_np = dev_of_node(scmi_dev); + struct device_node *cpu_np, *np; + struct device *cpu_dev; + int cpu, idx; + + if (!scmi_np) + return false; + + for_each_possible_cpu(cpu) { + cpu_dev = get_cpu_device(cpu); + if (!cpu_dev) + continue; + + cpu_np = dev_of_node(cpu_dev); + + np = of_parse_phandle(cpu_np, "clocks", 0); + of_node_put(np); + + if (np == scmi_np) + return true; + + idx = of_property_match_string(cpu_np, "power-domain-names", "perf"); + np = of_parse_phandle(cpu_np, "power-domains", idx); + of_node_put(np); + + if (np == scmi_np) + return true; + } + + /* + * Older Broadcom STB chips had a "clocks" property for CPU node(s) + * that did not match the SCMI performance protocol node, if we got + * there, it means we had such an older Device Tree, therefore return + * true to preserve backwards compatibility. + */ + if (of_machine_is_compatible("brcm,brcmstb")) + return true; + + return false; +} + +static int scmi_cpufreq_probe(struct scmi_device *sdev) +{ + int ret; + struct device *dev = &sdev->dev; + const struct scmi_handle *handle; + + handle = sdev->handle; + + if (!handle || !scmi_dev_used_by_cpus(dev)) + return -ENODEV; + + scmi_cpufreq_driver.driver_data = sdev; + + perf_ops = handle->devm_protocol_get(sdev, SCMI_PROTOCOL_PERF, &ph); + if (IS_ERR(perf_ops)) + return PTR_ERR(perf_ops); + +#ifdef CONFIG_COMMON_CLK + /* dummy clock provider as needed by OPP if clocks property is used */ + if (of_property_present(dev->of_node, "#clock-cells")) { + ret = devm_of_clk_add_hw_provider(dev, of_clk_hw_simple_get, NULL); + if (ret) + return dev_err_probe(dev, ret, "%s: registering clock provider failed\n", __func__); + } +#endif + + ret = cpufreq_register_driver(&scmi_cpufreq_driver); + if (ret) { + dev_err(dev, "%s: registering cpufreq failed, err: %d\n", + __func__, ret); + } + + return ret; +} + +static void scmi_cpufreq_remove(struct scmi_device *sdev) +{ + cpufreq_unregister_driver(&scmi_cpufreq_driver); +} + +static const struct scmi_device_id scmi_id_table[] = { + { SCMI_PROTOCOL_PERF, "cpufreq" }, + { }, +}; +MODULE_DEVICE_TABLE(scmi, scmi_id_table); + +static struct scmi_driver scmi_cpufreq_drv = { + .name = "scmi-cpufreq", + .probe = scmi_cpufreq_probe, + .remove = scmi_cpufreq_remove, + .id_table = scmi_id_table, +}; +module_scmi_driver(scmi_cpufreq_drv); + +MODULE_AUTHOR("Sudeep Holla <sudeep.holla@arm.com>"); +MODULE_DESCRIPTION("ARM SCMI CPUFreq interface driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/cpufreq/scpi-cpufreq.c b/drivers/cpufreq/scpi-cpufreq.c new file mode 100644 index 000000000000..e530345baddf --- /dev/null +++ b/drivers/cpufreq/scpi-cpufreq.c @@ -0,0 +1,234 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * System Control and Power Interface (SCPI) based CPUFreq Interface driver + * + * Copyright (C) 2015 ARM Ltd. + * Sudeep Holla <sudeep.holla@arm.com> + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/clk.h> +#include <linux/cpu.h> +#include <linux/cpufreq.h> +#include <linux/cpumask.h> +#include <linux/export.h> +#include <linux/module.h> +#include <linux/platform_device.h> +#include <linux/pm_opp.h> +#include <linux/scpi_protocol.h> +#include <linux/slab.h> +#include <linux/types.h> + +struct scpi_data { + struct clk *clk; + struct device *cpu_dev; +}; + +static struct scpi_ops *scpi_ops; + +static unsigned int scpi_cpufreq_get_rate(unsigned int cpu) +{ + struct cpufreq_policy *policy; + struct scpi_data *priv; + unsigned long rate; + + policy = cpufreq_cpu_get_raw(cpu); + if (unlikely(!policy)) + return 0; + + priv = policy->driver_data; + rate = clk_get_rate(priv->clk); + + return rate / 1000; +} + +static int +scpi_cpufreq_set_target(struct cpufreq_policy *policy, unsigned int index) +{ + unsigned long freq_khz = policy->freq_table[index].frequency; + struct scpi_data *priv = policy->driver_data; + unsigned long rate = freq_khz * 1000; + int ret; + + ret = clk_set_rate(priv->clk, rate); + + if (ret) + return ret; + + if (clk_get_rate(priv->clk) / 1000 != freq_khz) + return -EIO; + + return 0; +} + +static int +scpi_get_sharing_cpus(struct device *cpu_dev, struct cpumask *cpumask) +{ + int cpu, domain, tdomain; + struct device *tcpu_dev; + + domain = scpi_ops->device_domain_id(cpu_dev); + if (domain < 0) + return domain; + + for_each_present_cpu(cpu) { + if (cpu == cpu_dev->id) + continue; + + tcpu_dev = get_cpu_device(cpu); + if (!tcpu_dev) + continue; + + tdomain = scpi_ops->device_domain_id(tcpu_dev); + if (tdomain == domain) + cpumask_set_cpu(cpu, cpumask); + } + + return 0; +} + +static int scpi_cpufreq_init(struct cpufreq_policy *policy) +{ + int ret; + unsigned int latency; + struct device *cpu_dev; + struct scpi_data *priv; + struct cpufreq_frequency_table *freq_table; + + cpu_dev = get_cpu_device(policy->cpu); + if (!cpu_dev) { + pr_err("failed to get cpu%d device\n", policy->cpu); + return -ENODEV; + } + + ret = scpi_ops->add_opps_to_device(cpu_dev); + if (ret) { + dev_warn(cpu_dev, "failed to add opps to the device\n"); + return ret; + } + + ret = scpi_get_sharing_cpus(cpu_dev, policy->cpus); + if (ret) { + dev_warn(cpu_dev, "failed to get sharing cpumask\n"); + return ret; + } + + ret = dev_pm_opp_set_sharing_cpus(cpu_dev, policy->cpus); + if (ret) { + dev_err(cpu_dev, "%s: failed to mark OPPs as shared: %d\n", + __func__, ret); + return ret; + } + + ret = dev_pm_opp_get_opp_count(cpu_dev); + if (ret <= 0) { + dev_dbg(cpu_dev, "OPP table is not ready, deferring probe\n"); + ret = -EPROBE_DEFER; + goto out_free_opp; + } + + priv = kzalloc(sizeof(*priv), GFP_KERNEL); + if (!priv) { + ret = -ENOMEM; + goto out_free_opp; + } + + ret = dev_pm_opp_init_cpufreq_table(cpu_dev, &freq_table); + if (ret) { + dev_err(cpu_dev, "failed to init cpufreq table: %d\n", ret); + goto out_free_priv; + } + + priv->cpu_dev = cpu_dev; + priv->clk = clk_get(cpu_dev, NULL); + if (IS_ERR(priv->clk)) { + dev_err(cpu_dev, "%s: Failed to get clk for cpu: %d\n", + __func__, cpu_dev->id); + ret = PTR_ERR(priv->clk); + goto out_free_cpufreq_table; + } + + policy->driver_data = priv; + policy->freq_table = freq_table; + + /* scpi allows DVFS request for any domain from any CPU */ + policy->dvfs_possible_from_any_cpu = true; + + latency = scpi_ops->get_transition_latency(cpu_dev); + if (!latency) + latency = CPUFREQ_DEFAULT_TRANSITION_LATENCY_NS; + + policy->cpuinfo.transition_latency = latency; + + policy->fast_switch_possible = false; + + return 0; + +out_free_cpufreq_table: + dev_pm_opp_free_cpufreq_table(cpu_dev, &freq_table); +out_free_priv: + kfree(priv); +out_free_opp: + dev_pm_opp_remove_all_dynamic(cpu_dev); + + return ret; +} + +static void scpi_cpufreq_exit(struct cpufreq_policy *policy) +{ + struct scpi_data *priv = policy->driver_data; + + clk_put(priv->clk); + dev_pm_opp_free_cpufreq_table(priv->cpu_dev, &policy->freq_table); + dev_pm_opp_remove_all_dynamic(priv->cpu_dev); + kfree(priv); +} + +static struct cpufreq_driver scpi_cpufreq_driver = { + .name = "scpi-cpufreq", + .flags = CPUFREQ_HAVE_GOVERNOR_PER_POLICY | + CPUFREQ_NEED_INITIAL_FREQ_CHECK | + CPUFREQ_IS_COOLING_DEV, + .verify = cpufreq_generic_frequency_table_verify, + .get = scpi_cpufreq_get_rate, + .init = scpi_cpufreq_init, + .exit = scpi_cpufreq_exit, + .target_index = scpi_cpufreq_set_target, + .register_em = cpufreq_register_em_with_opp, +}; + +static int scpi_cpufreq_probe(struct platform_device *pdev) +{ + int ret; + + scpi_ops = get_scpi_ops(); + if (!scpi_ops) + return -EIO; + + ret = cpufreq_register_driver(&scpi_cpufreq_driver); + if (ret) + dev_err(&pdev->dev, "%s: registering cpufreq failed, err: %d\n", + __func__, ret); + return ret; +} + +static void scpi_cpufreq_remove(struct platform_device *pdev) +{ + cpufreq_unregister_driver(&scpi_cpufreq_driver); + scpi_ops = NULL; +} + +static struct platform_driver scpi_cpufreq_platdrv = { + .driver = { + .name = "scpi-cpufreq", + }, + .probe = scpi_cpufreq_probe, + .remove = scpi_cpufreq_remove, +}; +module_platform_driver(scpi_cpufreq_platdrv); + +MODULE_ALIAS("platform:scpi-cpufreq"); +MODULE_AUTHOR("Sudeep Holla <sudeep.holla@arm.com>"); +MODULE_DESCRIPTION("ARM SCPI CPUFreq interface driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/cpufreq/sh-cpufreq.c b/drivers/cpufreq/sh-cpufreq.c index 73adb64651e8..642ddb9ea217 100644 --- a/drivers/cpufreq/sh-cpufreq.c +++ b/drivers/cpufreq/sh-cpufreq.c @@ -23,79 +23,82 @@ #include <linux/cpumask.h> #include <linux/cpu.h> #include <linux/smp.h> -#include <linux/sched.h> /* set_cpus_allowed() */ #include <linux/clk.h> #include <linux/percpu.h> #include <linux/sh_clk.h> static DEFINE_PER_CPU(struct clk, sh_cpuclk); +struct cpufreq_target { + struct cpufreq_policy *policy; + unsigned int freq; +}; + static unsigned int sh_cpufreq_get(unsigned int cpu) { return (clk_get_rate(&per_cpu(sh_cpuclk, cpu)) + 500) / 1000; } -/* - * Here we notify other drivers of the proposed change and the final change. - */ -static int sh_cpufreq_target(struct cpufreq_policy *policy, - unsigned int target_freq, - unsigned int relation) +static long __sh_cpufreq_target(void *arg) { - unsigned int cpu = policy->cpu; + struct cpufreq_target *target = arg; + struct cpufreq_policy *policy = target->policy; + int cpu = policy->cpu; struct clk *cpuclk = &per_cpu(sh_cpuclk, cpu); - cpumask_t cpus_allowed; struct cpufreq_freqs freqs; struct device *dev; long freq; - cpus_allowed = current->cpus_allowed; - set_cpus_allowed_ptr(current, cpumask_of(cpu)); - - BUG_ON(smp_processor_id() != cpu); + if (smp_processor_id() != cpu) + return -ENODEV; dev = get_cpu_device(cpu); /* Convert target_freq from kHz to Hz */ - freq = clk_round_rate(cpuclk, target_freq * 1000); + freq = clk_round_rate(cpuclk, target->freq * 1000); if (freq < (policy->min * 1000) || freq > (policy->max * 1000)) return -EINVAL; - dev_dbg(dev, "requested frequency %u Hz\n", target_freq * 1000); + dev_dbg(dev, "requested frequency %u Hz\n", target->freq * 1000); freqs.old = sh_cpufreq_get(cpu); freqs.new = (freq + 500) / 1000; freqs.flags = 0; - cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE); - set_cpus_allowed_ptr(current, &cpus_allowed); + cpufreq_freq_transition_begin(target->policy, &freqs); clk_set_rate(cpuclk, freq); - cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); + cpufreq_freq_transition_end(target->policy, &freqs, 0); dev_dbg(dev, "set frequency %lu Hz\n", freq); - return 0; } -static int sh_cpufreq_verify(struct cpufreq_policy *policy) +/* + * Here we notify other drivers of the proposed change and the final change. + */ +static int sh_cpufreq_target(struct cpufreq_policy *policy, + unsigned int target_freq, + unsigned int relation) +{ + struct cpufreq_target data = { .policy = policy, .freq = target_freq }; + + return work_on_cpu(policy->cpu, __sh_cpufreq_target, &data); +} + +static int sh_cpufreq_verify(struct cpufreq_policy_data *policy) { struct clk *cpuclk = &per_cpu(sh_cpuclk, policy->cpu); - struct cpufreq_frequency_table *freq_table; - freq_table = cpuclk->nr_freqs ? cpuclk->freq_table : NULL; - if (freq_table) - return cpufreq_frequency_table_verify(policy, freq_table); + if (policy->freq_table) + return cpufreq_frequency_table_verify(policy); - cpufreq_verify_within_limits(policy, policy->cpuinfo.min_freq, - policy->cpuinfo.max_freq); + cpufreq_verify_within_cpu_limits(policy); policy->min = (clk_round_rate(cpuclk, 1) + 500) / 1000; policy->max = (clk_round_rate(cpuclk, ~0UL) + 500) / 1000; - cpufreq_verify_within_limits(policy, policy->cpuinfo.min_freq, - policy->cpuinfo.max_freq); - + cpufreq_verify_within_cpu_limits(policy); return 0; } @@ -114,15 +117,9 @@ static int sh_cpufreq_cpu_init(struct cpufreq_policy *policy) return PTR_ERR(cpuclk); } - policy->cur = sh_cpufreq_get(cpu); - freq_table = cpuclk->nr_freqs ? cpuclk->freq_table : NULL; if (freq_table) { - int result; - - result = cpufreq_frequency_table_cpuinfo(policy, freq_table); - if (!result) - cpufreq_frequency_table_get_attr(freq_table, cpu); + policy->freq_table = freq_table; } else { dev_notice(dev, "no frequency table found, falling back " "to rate rounding.\n"); @@ -133,41 +130,25 @@ static int sh_cpufreq_cpu_init(struct cpufreq_policy *policy) (clk_round_rate(cpuclk, ~0UL) + 500) / 1000; } - policy->cpuinfo.transition_latency = CPUFREQ_ETERNAL; - - dev_info(dev, "CPU Frequencies - Minimum %u.%03u MHz, " - "Maximum %u.%03u MHz.\n", - policy->min / 1000, policy->min % 1000, - policy->max / 1000, policy->max % 1000); - return 0; } -static int sh_cpufreq_cpu_exit(struct cpufreq_policy *policy) +static void sh_cpufreq_cpu_exit(struct cpufreq_policy *policy) { unsigned int cpu = policy->cpu; struct clk *cpuclk = &per_cpu(sh_cpuclk, cpu); - cpufreq_frequency_table_put_attr(cpu); clk_put(cpuclk); - - return 0; } -static struct freq_attr *sh_freq_attr[] = { - &cpufreq_freq_attr_scaling_available_freqs, - NULL, -}; - static struct cpufreq_driver sh_cpufreq_driver = { - .owner = THIS_MODULE, .name = "sh", + .flags = CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING, .get = sh_cpufreq_get, .target = sh_cpufreq_target, .verify = sh_cpufreq_verify, .init = sh_cpufreq_cpu_init, .exit = sh_cpufreq_cpu_exit, - .attr = sh_freq_attr, }; static int __init sh_cpufreq_module_init(void) diff --git a/drivers/cpufreq/sparc-us2e-cpufreq.c b/drivers/cpufreq/sparc-us2e-cpufreq.c index 93061a408773..15899dd77c08 100644 --- a/drivers/cpufreq/sparc-us2e-cpufreq.c +++ b/drivers/cpufreq/sparc-us2e-cpufreq.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-only /* us2e_cpufreq.c: UltraSPARC-IIe cpu frequency support * * Copyright (C) 2003 David S. Miller (davem@redhat.com) @@ -19,8 +20,6 @@ #include <asm/asi.h> #include <asm/timer.h> -static struct cpufreq_driver *cpufreq_us2e_driver; - struct us2e_freq_percpu_info { struct cpufreq_frequency_table table[6]; }; @@ -118,10 +117,6 @@ static void us2e_transition(unsigned long estar, unsigned long new_bits, unsigned long clock_tick, unsigned long old_divisor, unsigned long divisor) { - unsigned long flags; - - local_irq_save(flags); - estar &= ~ESTAR_MODE_DIV_MASK; /* This is based upon the state transition diagram in the IIe manual. */ @@ -152,8 +147,6 @@ static void us2e_transition(unsigned long estar, unsigned long new_bits, } else { BUG(); } - - local_irq_restore(flags); } static unsigned long index_to_estar_mode(unsigned int index) @@ -229,79 +222,54 @@ static unsigned long estar_to_divisor(unsigned long estar) return ret; } +static void __us2e_freq_get(void *arg) +{ + unsigned long *estar = arg; + + *estar = read_hbreg(HBIRD_ESTAR_MODE_ADDR); +} + static unsigned int us2e_freq_get(unsigned int cpu) { - cpumask_t cpus_allowed; unsigned long clock_tick, estar; - cpumask_copy(&cpus_allowed, tsk_cpus_allowed(current)); - set_cpus_allowed_ptr(current, cpumask_of(cpu)); - clock_tick = sparc64_get_clock_tick(cpu) / 1000; - estar = read_hbreg(HBIRD_ESTAR_MODE_ADDR); - - set_cpus_allowed_ptr(current, &cpus_allowed); + if (smp_call_function_single(cpu, __us2e_freq_get, &estar, 1)) + return 0; return clock_tick / estar_to_divisor(estar); } -static void us2e_set_cpu_divider_index(struct cpufreq_policy *policy, - unsigned int index) +static void __us2e_freq_target(void *arg) { - unsigned int cpu = policy->cpu; + unsigned int cpu = smp_processor_id(); + unsigned int *index = arg; unsigned long new_bits, new_freq; unsigned long clock_tick, divisor, old_divisor, estar; - cpumask_t cpus_allowed; - struct cpufreq_freqs freqs; - - cpumask_copy(&cpus_allowed, tsk_cpus_allowed(current)); - set_cpus_allowed_ptr(current, cpumask_of(cpu)); new_freq = clock_tick = sparc64_get_clock_tick(cpu) / 1000; - new_bits = index_to_estar_mode(index); - divisor = index_to_divisor(index); + new_bits = index_to_estar_mode(*index); + divisor = index_to_divisor(*index); new_freq /= divisor; estar = read_hbreg(HBIRD_ESTAR_MODE_ADDR); old_divisor = estar_to_divisor(estar); - freqs.old = clock_tick / old_divisor; - freqs.new = new_freq; - cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE); - - if (old_divisor != divisor) + if (old_divisor != divisor) { us2e_transition(estar, new_bits, clock_tick * 1000, old_divisor, divisor); - - cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); - - set_cpus_allowed_ptr(current, &cpus_allowed); + } } -static int us2e_freq_target(struct cpufreq_policy *policy, - unsigned int target_freq, - unsigned int relation) +static int us2e_freq_target(struct cpufreq_policy *policy, unsigned int index) { - unsigned int new_index = 0; - - if (cpufreq_frequency_table_target(policy, - &us2e_freq_table[policy->cpu].table[0], - target_freq, relation, &new_index)) - return -EINVAL; - - us2e_set_cpu_divider_index(policy, new_index); - - return 0; -} + unsigned int cpu = policy->cpu; -static int us2e_freq_verify(struct cpufreq_policy *policy) -{ - return cpufreq_frequency_table_verify(policy, - &us2e_freq_table[policy->cpu].table[0]); + return smp_call_function_single(cpu, __us2e_freq_target, &index, 1); } -static int __init us2e_freq_cpu_init(struct cpufreq_policy *policy) +static int us2e_freq_cpu_init(struct cpufreq_policy *policy) { unsigned int cpu = policy->cpu; unsigned long clock_tick = sparc64_get_clock_tick(cpu) / 1000; @@ -323,18 +291,25 @@ static int __init us2e_freq_cpu_init(struct cpufreq_policy *policy) policy->cpuinfo.transition_latency = 0; policy->cur = clock_tick; + policy->freq_table = table; - return cpufreq_frequency_table_cpuinfo(policy, table); + return 0; } -static int us2e_freq_cpu_exit(struct cpufreq_policy *policy) +static void us2e_freq_cpu_exit(struct cpufreq_policy *policy) { - if (cpufreq_us2e_driver) - us2e_set_cpu_divider_index(policy, 0); - - return 0; + us2e_freq_target(policy, 0); } +static struct cpufreq_driver cpufreq_us2e_driver = { + .name = "UltraSPARC-IIe", + .init = us2e_freq_cpu_init, + .verify = cpufreq_generic_frequency_table_verify, + .target_index = us2e_freq_target, + .get = us2e_freq_get, + .exit = us2e_freq_cpu_exit, +}; + static int __init us2e_freq_init(void) { unsigned long manuf, impl, ver; @@ -348,41 +323,15 @@ static int __init us2e_freq_init(void) impl = ((ver >> 32) & 0xffff); if (manuf == 0x17 && impl == 0x13) { - struct cpufreq_driver *driver; - - ret = -ENOMEM; - driver = kzalloc(sizeof(struct cpufreq_driver), GFP_KERNEL); - if (!driver) - goto err_out; - - us2e_freq_table = kzalloc( - (NR_CPUS * sizeof(struct us2e_freq_percpu_info)), - GFP_KERNEL); + us2e_freq_table = kcalloc(NR_CPUS, sizeof(*us2e_freq_table), + GFP_KERNEL); if (!us2e_freq_table) - goto err_out; - - driver->init = us2e_freq_cpu_init; - driver->verify = us2e_freq_verify; - driver->target = us2e_freq_target; - driver->get = us2e_freq_get; - driver->exit = us2e_freq_cpu_exit; - driver->owner = THIS_MODULE, - strcpy(driver->name, "UltraSPARC-IIe"); - - cpufreq_us2e_driver = driver; - ret = cpufreq_register_driver(driver); - if (ret) - goto err_out; + return -ENOMEM; - return 0; + ret = cpufreq_register_driver(&cpufreq_us2e_driver); + if (ret) + kfree(us2e_freq_table); -err_out: - if (driver) { - kfree(driver); - cpufreq_us2e_driver = NULL; - } - kfree(us2e_freq_table); - us2e_freq_table = NULL; return ret; } @@ -391,13 +340,8 @@ err_out: static void __exit us2e_freq_exit(void) { - if (cpufreq_us2e_driver) { - cpufreq_unregister_driver(cpufreq_us2e_driver); - kfree(cpufreq_us2e_driver); - cpufreq_us2e_driver = NULL; - kfree(us2e_freq_table); - us2e_freq_table = NULL; - } + cpufreq_unregister_driver(&cpufreq_us2e_driver); + kfree(us2e_freq_table); } MODULE_AUTHOR("David S. Miller <davem@redhat.com>"); diff --git a/drivers/cpufreq/sparc-us3-cpufreq.c b/drivers/cpufreq/sparc-us3-cpufreq.c index 880ee293d61e..de50a2f3b124 100644 --- a/drivers/cpufreq/sparc-us3-cpufreq.c +++ b/drivers/cpufreq/sparc-us3-cpufreq.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-only /* us3_cpufreq.c: UltraSPARC-III cpu frequency support * * Copyright (C) 2003 David S. Miller (davem@redhat.com) @@ -18,8 +19,6 @@ #include <asm/head.h> #include <asm/timer.h> -static struct cpufreq_driver *cpufreq_us3_driver; - struct us3_freq_percpu_info { struct cpufreq_frequency_table table[4]; }; @@ -35,22 +34,28 @@ static struct us3_freq_percpu_info *us3_freq_table; #define SAFARI_CFG_DIV_32 0x0000000080000000UL #define SAFARI_CFG_DIV_MASK 0x00000000C0000000UL -static unsigned long read_safari_cfg(void) +static void read_safari_cfg(void *arg) { - unsigned long ret; + unsigned long ret, *val = arg; __asm__ __volatile__("ldxa [%%g0] %1, %0" : "=&r" (ret) : "i" (ASI_SAFARI_CONFIG)); - return ret; + *val = ret; } -static void write_safari_cfg(unsigned long val) +static void update_safari_cfg(void *arg) { + unsigned long reg, *new_bits = arg; + + read_safari_cfg(®); + reg &= ~SAFARI_CFG_DIV_MASK; + reg |= *new_bits; + __asm__ __volatile__("stxa %0, [%%g0] %1\n\t" "membar #Sync" : /* no outputs */ - : "r" (val), "i" (ASI_SAFARI_CONFIG) + : "r" (reg), "i" (ASI_SAFARI_CONFIG) : "memory"); } @@ -78,31 +83,17 @@ static unsigned long get_current_freq(unsigned int cpu, unsigned long safari_cfg static unsigned int us3_freq_get(unsigned int cpu) { - cpumask_t cpus_allowed; unsigned long reg; - unsigned int ret; - - cpumask_copy(&cpus_allowed, tsk_cpus_allowed(current)); - set_cpus_allowed_ptr(current, cpumask_of(cpu)); - - reg = read_safari_cfg(); - ret = get_current_freq(cpu, reg); - set_cpus_allowed_ptr(current, &cpus_allowed); - - return ret; + if (smp_call_function_single(cpu, read_safari_cfg, ®, 1)) + return 0; + return get_current_freq(cpu, reg); } -static void us3_set_cpu_divider_index(struct cpufreq_policy *policy, - unsigned int index) +static int us3_freq_target(struct cpufreq_policy *policy, unsigned int index) { unsigned int cpu = policy->cpu; - unsigned long new_bits, new_freq, reg; - cpumask_t cpus_allowed; - struct cpufreq_freqs freqs; - - cpumask_copy(&cpus_allowed, tsk_cpus_allowed(current)); - set_cpus_allowed_ptr(current, cpumask_of(cpu)); + unsigned long new_bits, new_freq; new_freq = sparc64_get_clock_tick(cpu) / 1000; switch (index) { @@ -123,46 +114,10 @@ static void us3_set_cpu_divider_index(struct cpufreq_policy *policy, BUG(); } - reg = read_safari_cfg(); - - freqs.old = get_current_freq(cpu, reg); - freqs.new = new_freq; - cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE); - - reg &= ~SAFARI_CFG_DIV_MASK; - reg |= new_bits; - write_safari_cfg(reg); - - cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); - - set_cpus_allowed_ptr(current, &cpus_allowed); -} - -static int us3_freq_target(struct cpufreq_policy *policy, - unsigned int target_freq, - unsigned int relation) -{ - unsigned int new_index = 0; - - if (cpufreq_frequency_table_target(policy, - &us3_freq_table[policy->cpu].table[0], - target_freq, - relation, - &new_index)) - return -EINVAL; - - us3_set_cpu_divider_index(policy, new_index); - - return 0; + return smp_call_function_single(cpu, update_safari_cfg, &new_bits, 1); } -static int us3_freq_verify(struct cpufreq_policy *policy) -{ - return cpufreq_frequency_table_verify(policy, - &us3_freq_table[policy->cpu].table[0]); -} - -static int __init us3_freq_cpu_init(struct cpufreq_policy *policy) +static int us3_freq_cpu_init(struct cpufreq_policy *policy) { unsigned int cpu = policy->cpu; unsigned long clock_tick = sparc64_get_clock_tick(cpu) / 1000; @@ -180,18 +135,25 @@ static int __init us3_freq_cpu_init(struct cpufreq_policy *policy) policy->cpuinfo.transition_latency = 0; policy->cur = clock_tick; + policy->freq_table = table; - return cpufreq_frequency_table_cpuinfo(policy, table); + return 0; } -static int us3_freq_cpu_exit(struct cpufreq_policy *policy) +static void us3_freq_cpu_exit(struct cpufreq_policy *policy) { - if (cpufreq_us3_driver) - us3_set_cpu_divider_index(policy, 0); - - return 0; + us3_freq_target(policy, 0); } +static struct cpufreq_driver cpufreq_us3_driver = { + .name = "UltraSPARC-III", + .init = us3_freq_cpu_init, + .verify = cpufreq_generic_frequency_table_verify, + .target_index = us3_freq_target, + .get = us3_freq_get, + .exit = us3_freq_cpu_exit, +}; + static int __init us3_freq_init(void) { unsigned long manuf, impl, ver; @@ -209,41 +171,15 @@ static int __init us3_freq_init(void) impl == CHEETAH_PLUS_IMPL || impl == JAGUAR_IMPL || impl == PANTHER_IMPL)) { - struct cpufreq_driver *driver; - - ret = -ENOMEM; - driver = kzalloc(sizeof(struct cpufreq_driver), GFP_KERNEL); - if (!driver) - goto err_out; - - us3_freq_table = kzalloc( - (NR_CPUS * sizeof(struct us3_freq_percpu_info)), - GFP_KERNEL); + us3_freq_table = kcalloc(NR_CPUS, sizeof(*us3_freq_table), + GFP_KERNEL); if (!us3_freq_table) - goto err_out; - - driver->init = us3_freq_cpu_init; - driver->verify = us3_freq_verify; - driver->target = us3_freq_target; - driver->get = us3_freq_get; - driver->exit = us3_freq_cpu_exit; - driver->owner = THIS_MODULE, - strcpy(driver->name, "UltraSPARC-III"); - - cpufreq_us3_driver = driver; - ret = cpufreq_register_driver(driver); - if (ret) - goto err_out; + return -ENOMEM; - return 0; + ret = cpufreq_register_driver(&cpufreq_us3_driver); + if (ret) + kfree(us3_freq_table); -err_out: - if (driver) { - kfree(driver); - cpufreq_us3_driver = NULL; - } - kfree(us3_freq_table); - us3_freq_table = NULL; return ret; } @@ -252,13 +188,8 @@ err_out: static void __exit us3_freq_exit(void) { - if (cpufreq_us3_driver) { - cpufreq_unregister_driver(cpufreq_us3_driver); - kfree(cpufreq_us3_driver); - cpufreq_us3_driver = NULL; - kfree(us3_freq_table); - us3_freq_table = NULL; - } + cpufreq_unregister_driver(&cpufreq_us3_driver); + kfree(us3_freq_table); } MODULE_AUTHOR("David S. Miller <davem@redhat.com>"); diff --git a/drivers/cpufreq/spear-cpufreq.c b/drivers/cpufreq/spear-cpufreq.c index c3efa7f2a908..2a1550e1aa21 100644 --- a/drivers/cpufreq/spear-cpufreq.c +++ b/drivers/cpufreq/spear-cpufreq.c @@ -19,6 +19,7 @@ #include <linux/init.h> #include <linux/module.h> #include <linux/of.h> +#include <linux/platform_device.h> #include <linux/slab.h> #include <linux/types.h> @@ -30,16 +31,6 @@ static struct { u32 cnt; } spear_cpufreq; -static int spear_cpufreq_verify(struct cpufreq_policy *policy) -{ - return cpufreq_frequency_table_verify(policy, spear_cpufreq.freq_tbl); -} - -static unsigned int spear_cpufreq_get(unsigned int cpu) -{ - return clk_get_rate(spear_cpufreq.clk) / 1000; -} - static struct clk *spear1340_cpu_get_possible_parent(unsigned long newfreq) { struct clk *sys_pclk; @@ -48,7 +39,7 @@ static struct clk *spear1340_cpu_get_possible_parent(unsigned long newfreq) * In SPEAr1340, cpu clk's parent sys clk can take input from * following sources */ - const char *sys_clk_src[] = { + static const char * const sys_clk_src[] = { "sys_syn_clk", "pll1_clk", "pll2_clk", @@ -110,20 +101,14 @@ static int spear1340_set_cpu_rate(struct clk *sys_pclk, unsigned long newfreq) } static int spear_cpufreq_target(struct cpufreq_policy *policy, - unsigned int target_freq, unsigned int relation) + unsigned int index) { - struct cpufreq_freqs freqs; - unsigned long newfreq; + long newfreq; struct clk *srcclk; - int index, ret, mult = 1; - - if (cpufreq_frequency_table_target(policy, spear_cpufreq.freq_tbl, - target_freq, relation, &index)) - return -EINVAL; - - freqs.old = spear_cpufreq_get(0); + int ret, mult = 1; newfreq = spear_cpufreq.freq_tbl[index].frequency * 1000; + if (of_machine_is_compatible("st,spear1340")) { /* * SPEAr1340 is special in the sense that due to the possibility @@ -149,113 +134,73 @@ static int spear_cpufreq_target(struct cpufreq_policy *policy, } newfreq = clk_round_rate(srcclk, newfreq * mult); - if (newfreq < 0) { + if (newfreq <= 0) { pr_err("clk_round_rate failed for cpu src clock\n"); return newfreq; } - freqs.new = newfreq / 1000; - freqs.new /= mult; - - cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE); - if (mult == 2) ret = spear1340_set_cpu_rate(srcclk, newfreq); else ret = clk_set_rate(spear_cpufreq.clk, newfreq); - /* Get current rate after clk_set_rate, in case of failure */ - if (ret) { + if (ret) pr_err("CPU Freq: cpu clk_set_rate failed: %d\n", ret); - freqs.new = clk_get_rate(spear_cpufreq.clk) / 1000; - } - cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); return ret; } static int spear_cpufreq_init(struct cpufreq_policy *policy) { - int ret; - - ret = cpufreq_frequency_table_cpuinfo(policy, spear_cpufreq.freq_tbl); - if (ret) { - pr_err("cpufreq_frequency_table_cpuinfo() failed"); - return ret; - } - - cpufreq_frequency_table_get_attr(spear_cpufreq.freq_tbl, policy->cpu); - policy->cpuinfo.transition_latency = spear_cpufreq.transition_latency; - policy->cur = spear_cpufreq_get(0); - - cpumask_setall(policy->cpus); - - return 0; -} - -static int spear_cpufreq_exit(struct cpufreq_policy *policy) -{ - cpufreq_frequency_table_put_attr(policy->cpu); + policy->clk = spear_cpufreq.clk; + cpufreq_generic_init(policy, spear_cpufreq.freq_tbl, + spear_cpufreq.transition_latency); return 0; } -static struct freq_attr *spear_cpufreq_attr[] = { - &cpufreq_freq_attr_scaling_available_freqs, - NULL, -}; - static struct cpufreq_driver spear_cpufreq_driver = { .name = "cpufreq-spear", - .flags = CPUFREQ_STICKY, - .verify = spear_cpufreq_verify, - .target = spear_cpufreq_target, - .get = spear_cpufreq_get, + .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK, + .verify = cpufreq_generic_frequency_table_verify, + .target_index = spear_cpufreq_target, + .get = cpufreq_generic_get, .init = spear_cpufreq_init, - .exit = spear_cpufreq_exit, - .attr = spear_cpufreq_attr, }; -static int spear_cpufreq_driver_init(void) +static int spear_cpufreq_probe(struct platform_device *pdev) { struct device_node *np; - const struct property *prop; struct cpufreq_frequency_table *freq_tbl; - const __be32 *val; - int cnt, i, ret; + u32 val; + int cnt, ret, i = 0; - np = of_find_node_by_path("/cpus/cpu@0"); + np = of_cpu_device_node_get(0); if (!np) { - pr_err("No cpu node found"); + pr_err("No cpu node found\n"); return -ENODEV; } if (of_property_read_u32(np, "clock-latency", &spear_cpufreq.transition_latency)) - spear_cpufreq.transition_latency = CPUFREQ_ETERNAL; + spear_cpufreq.transition_latency = CPUFREQ_DEFAULT_TRANSITION_LATENCY_NS; - prop = of_find_property(np, "cpufreq_tbl", NULL); - if (!prop || !prop->value) { - pr_err("Invalid cpufreq_tbl"); + cnt = of_property_count_u32_elems(np, "cpufreq_tbl"); + if (cnt <= 0) { + pr_err("Invalid cpufreq_tbl\n"); ret = -ENODEV; goto out_put_node; } - cnt = prop->length / sizeof(u32); - val = prop->value; - - freq_tbl = kmalloc(sizeof(*freq_tbl) * (cnt + 1), GFP_KERNEL); + freq_tbl = kcalloc(cnt + 1, sizeof(*freq_tbl), GFP_KERNEL); if (!freq_tbl) { ret = -ENOMEM; goto out_put_node; } - for (i = 0; i < cnt; i++) { - freq_tbl[i].driver_data = i; - freq_tbl[i].frequency = be32_to_cpup(val++); - } + of_property_for_each_u32(np, "cpufreq_tbl", val) + freq_tbl[i++].frequency = val; - freq_tbl[i].driver_data = i; - freq_tbl[i].frequency = CPUFREQ_TABLE_END; + freq_tbl[cnt].frequency = CPUFREQ_TABLE_END; spear_cpufreq.freq_tbl = freq_tbl; @@ -283,7 +228,14 @@ out_put_node: of_node_put(np); return ret; } -late_initcall(spear_cpufreq_driver_init); + +static struct platform_driver spear_cpufreq_platdrv = { + .driver = { + .name = "spear-cpufreq", + }, + .probe = spear_cpufreq_probe, +}; +module_platform_driver(spear_cpufreq_platdrv); MODULE_AUTHOR("Deepak Sikri <deepak.sikri@st.com>"); MODULE_DESCRIPTION("SPEAr CPUFreq driver"); diff --git a/drivers/cpufreq/speedstep-centrino.c b/drivers/cpufreq/speedstep-centrino.c index 0915e712fbdc..3e6e85a92212 100644 --- a/drivers/cpufreq/speedstep-centrino.c +++ b/drivers/cpufreq/speedstep-centrino.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * cpufreq driver for Enhanced SpeedStep, as found in Intel's Pentium * M (part of the Centrino chipset). @@ -13,6 +14,8 @@ * Copyright (C) 2003 Jeremy Fitzhardinge <jeremy@goop.org> */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + #include <linux/kernel.h> #include <linux/module.h> #include <linux/init.h> @@ -27,8 +30,7 @@ #include <asm/cpufeature.h> #include <asm/cpu_device_id.h> -#define PFX "speedstep-centrino: " -#define MAINTAINER "cpufreq@vger.kernel.org" +#define MAINTAINER "linux-pm@vger.kernel.org" #define INTEL_MSR_RANGE (0xffff) @@ -36,7 +38,7 @@ struct cpu_id { __u8 x86; /* CPU family */ __u8 x86_model; /* model */ - __u8 x86_mask; /* stepping */ + __u8 x86_stepping; /* stepping */ }; enum { @@ -276,7 +278,7 @@ static int centrino_verify_cpu_id(const struct cpuinfo_x86 *c, { if ((c->x86 == x->x86) && (c->x86_model == x->x86_model) && - (c->x86_mask == x->x86_mask)) + (c->x86_stepping == x->x86_stepping)) return 1; return 0; } @@ -343,9 +345,7 @@ static unsigned int get_cur_freq(unsigned int cpu) static int centrino_cpu_init(struct cpufreq_policy *policy) { struct cpuinfo_x86 *cpu = &cpu_data(policy->cpu); - unsigned freq; unsigned l, h; - int ret; int i; /* Only Intel makes Enhanced Speedstep-capable CPUs */ @@ -373,9 +373,8 @@ static int centrino_cpu_init(struct cpufreq_policy *policy) return -ENODEV; } - if (centrino_cpu_init_table(policy)) { + if (centrino_cpu_init_table(policy)) return -ENODEV; - } /* Check to see if Enhanced SpeedStep is enabled, and try to enable it if not. */ @@ -389,75 +388,39 @@ static int centrino_cpu_init(struct cpufreq_policy *policy) /* check to see if it stuck */ rdmsr(MSR_IA32_MISC_ENABLE, l, h); if (!(l & MSR_IA32_MISC_ENABLE_ENHANCED_SPEEDSTEP)) { - printk(KERN_INFO PFX - "couldn't enable Enhanced SpeedStep\n"); + pr_info("couldn't enable Enhanced SpeedStep\n"); return -ENODEV; } } - freq = get_cur_freq(policy->cpu); policy->cpuinfo.transition_latency = 10000; /* 10uS transition latency */ - policy->cur = freq; - - pr_debug("centrino_cpu_init: cur=%dkHz\n", policy->cur); - - ret = cpufreq_frequency_table_cpuinfo(policy, - per_cpu(centrino_model, policy->cpu)->op_points); - if (ret) - return (ret); - - cpufreq_frequency_table_get_attr( - per_cpu(centrino_model, policy->cpu)->op_points, policy->cpu); + policy->freq_table = per_cpu(centrino_model, policy->cpu)->op_points; return 0; } -static int centrino_cpu_exit(struct cpufreq_policy *policy) +static void centrino_cpu_exit(struct cpufreq_policy *policy) { unsigned int cpu = policy->cpu; - if (!per_cpu(centrino_model, cpu)) - return -ENODEV; - - cpufreq_frequency_table_put_attr(cpu); - - per_cpu(centrino_model, cpu) = NULL; - - return 0; + if (per_cpu(centrino_model, cpu)) + per_cpu(centrino_model, cpu) = NULL; } /** - * centrino_verify - verifies a new CPUFreq policy + * centrino_target - set a new CPUFreq policy * @policy: new policy - * - * Limit must be within this model's frequency range at least one - * border included. - */ -static int centrino_verify (struct cpufreq_policy *policy) -{ - return cpufreq_frequency_table_verify(policy, - per_cpu(centrino_model, policy->cpu)->op_points); -} - -/** - * centrino_setpolicy - set a new CPUFreq policy - * @policy: new policy - * @target_freq: the target frequency - * @relation: how that frequency relates to achieved frequency - * (CPUFREQ_RELATION_L or CPUFREQ_RELATION_H) + * @index: index of target frequency * * Sets a new CPUFreq policy. */ -static int centrino_target (struct cpufreq_policy *policy, - unsigned int target_freq, - unsigned int relation) +static int centrino_target(struct cpufreq_policy *policy, unsigned int index) { - unsigned int newstate = 0; unsigned int msr, oldmsr = 0, h = 0, cpu = policy->cpu; - struct cpufreq_freqs freqs; int retval = 0; - unsigned int j, first_cpu, tmp; + unsigned int j, first_cpu; + struct cpufreq_frequency_table *op_points; cpumask_var_t covered_cpus; if (unlikely(!zalloc_cpumask_var(&covered_cpus, GFP_KERNEL))) @@ -468,16 +431,8 @@ static int centrino_target (struct cpufreq_policy *policy, goto out; } - if (unlikely(cpufreq_frequency_table_target(policy, - per_cpu(centrino_model, cpu)->op_points, - target_freq, - relation, - &newstate))) { - retval = -EINVAL; - goto out; - } - first_cpu = 1; + op_points = &per_cpu(centrino_model, cpu)->op_points[index]; for_each_cpu(j, policy->cpus) { int good_cpu; @@ -501,7 +456,7 @@ static int centrino_target (struct cpufreq_policy *policy, break; } - msr = per_cpu(centrino_model, cpu)->op_points[newstate].driver_data; + msr = op_points->driver_data; if (first_cpu) { rdmsr_on_cpu(good_cpu, MSR_IA32_PERF_CTL, &oldmsr, &h); @@ -512,15 +467,6 @@ static int centrino_target (struct cpufreq_policy *policy, goto out; } - freqs.old = extract_clock(oldmsr, cpu, 0); - freqs.new = extract_clock(msr, cpu, 0); - - pr_debug("target=%dkHz old=%d new=%d msr=%04x\n", - target_freq, freqs.old, freqs.new, msr); - - cpufreq_notify_transition(policy, &freqs, - CPUFREQ_PRECHANGE); - first_cpu = 0; /* all but 16 LSB are reserved, treat them with care */ oldmsr &= ~0xffff; @@ -535,8 +481,6 @@ static int centrino_target (struct cpufreq_policy *policy, cpumask_set_cpu(j, covered_cpus); } - cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); - if (unlikely(retval)) { /* * We have failed halfway through the frequency change. @@ -547,12 +491,6 @@ static int centrino_target (struct cpufreq_policy *policy, for_each_cpu(j, covered_cpus) wrmsr_on_cpu(j, MSR_IA32_PERF_CTL, oldmsr, h); - - tmp = freqs.new; - freqs.new = freqs.old; - freqs.old = tmp; - cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE); - cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); } retval = 0; @@ -561,21 +499,14 @@ out: return retval; } -static struct freq_attr* centrino_attr[] = { - &cpufreq_freq_attr_scaling_available_freqs, - NULL, -}; - static struct cpufreq_driver centrino_driver = { .name = "centrino", /* should be speedstep-centrino, but there's a 16 char limit */ .init = centrino_cpu_init, .exit = centrino_cpu_exit, - .verify = centrino_verify, - .target = centrino_target, + .verify = cpufreq_generic_frequency_table_verify, + .target_index = centrino_target, .get = get_cur_freq, - .attr = centrino_attr, - .owner = THIS_MODULE, }; /* @@ -584,18 +515,12 @@ static struct cpufreq_driver centrino_driver = { * or ASCII model IDs. */ static const struct x86_cpu_id centrino_ids[] = { - { X86_VENDOR_INTEL, 6, 9, X86_FEATURE_EST }, - { X86_VENDOR_INTEL, 6, 13, X86_FEATURE_EST }, - { X86_VENDOR_INTEL, 6, 13, X86_FEATURE_EST }, - { X86_VENDOR_INTEL, 6, 13, X86_FEATURE_EST }, - { X86_VENDOR_INTEL, 15, 3, X86_FEATURE_EST }, - { X86_VENDOR_INTEL, 15, 4, X86_FEATURE_EST }, + X86_MATCH_VFM_FEATURE(IFM( 6, 9), X86_FEATURE_EST, NULL), + X86_MATCH_VFM_FEATURE(IFM( 6, 13), X86_FEATURE_EST, NULL), + X86_MATCH_VFM_FEATURE(IFM(15, 3), X86_FEATURE_EST, NULL), + X86_MATCH_VFM_FEATURE(IFM(15, 4), X86_FEATURE_EST, NULL), {} }; -#if 0 -/* Autoload or not? Do not for now. */ -MODULE_DEVICE_TABLE(x86cpu, centrino_ids); -#endif /** * centrino_init - initializes the Enhanced SpeedStep CPUFreq driver diff --git a/drivers/cpufreq/speedstep-ich.c b/drivers/cpufreq/speedstep-ich.c index e2e5aa971452..262cfbde9ca7 100644 --- a/drivers/cpufreq/speedstep-ich.c +++ b/drivers/cpufreq/speedstep-ich.c @@ -1,8 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * (C) 2001 Dave Jones, Arjan van de ven. * (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de> * - * Licensed under the terms of the GNU GPL License version 2. * Based upon reverse engineered information, and on Intel documentation * for chipsets ICH2-M and ICH3-M. * @@ -18,6 +18,8 @@ * SPEEDSTEP - DEFINITIONS * *********************************************************************/ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + #include <linux/kernel.h> #include <linux/module.h> #include <linux/init.h> @@ -49,9 +51,9 @@ static u32 pmbase; * are in kHz for the time being. */ static struct cpufreq_frequency_table speedstep_freqs[] = { - {SPEEDSTEP_HIGH, 0}, - {SPEEDSTEP_LOW, 0}, - {0, CPUFREQ_TABLE_END}, + {0, SPEEDSTEP_HIGH, 0}, + {0, SPEEDSTEP_LOW, 0}, + {0, 0, CPUFREQ_TABLE_END}, }; @@ -68,13 +70,13 @@ static int speedstep_find_register(void) /* get PMBASE */ pci_read_config_dword(speedstep_chipset_dev, 0x40, &pmbase); if (!(pmbase & 0x01)) { - printk(KERN_ERR "speedstep-ich: could not find speedstep register\n"); + pr_err("could not find speedstep register\n"); return -ENODEV; } pmbase &= 0xFFFFFFFE; if (!pmbase) { - printk(KERN_ERR "speedstep-ich: could not find speedstep register\n"); + pr_err("could not find speedstep register\n"); return -ENODEV; } @@ -136,7 +138,7 @@ static void speedstep_set_state(unsigned int state) pr_debug("change to %u MHz succeeded\n", speedstep_get_frequency(speedstep_processor) / 1000); else - printk(KERN_ERR "cpufreq: change failed - I/O error\n"); + pr_err("change failed - I/O error\n"); return; } @@ -205,7 +207,7 @@ static unsigned int speedstep_detect_chipset(void) * 8100 which use a pretty old revision of the 82815 * host bridge. Abort on these systems. */ - static struct pci_dev *hostbridge; + struct pci_dev *hostbridge; hostbridge = pci_get_subsys(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82815_MC, @@ -241,8 +243,7 @@ static unsigned int speedstep_get(unsigned int cpu) unsigned int speed; /* You're supposed to ensure CPU is online. */ - if (smp_call_function_single(cpu, get_freq_data, &speed, 1) != 0) - BUG(); + BUG_ON(smp_call_function_single(cpu, get_freq_data, &speed, 1)); pr_debug("detected %u kHz as current frequency\n", speed); return speed; @@ -251,56 +252,23 @@ static unsigned int speedstep_get(unsigned int cpu) /** * speedstep_target - set a new CPUFreq policy * @policy: new policy - * @target_freq: the target frequency - * @relation: how that frequency relates to achieved frequency - * (CPUFREQ_RELATION_L or CPUFREQ_RELATION_H) + * @index: index of target frequency * * Sets a new CPUFreq policy. */ -static int speedstep_target(struct cpufreq_policy *policy, - unsigned int target_freq, - unsigned int relation) +static int speedstep_target(struct cpufreq_policy *policy, unsigned int index) { - unsigned int newstate = 0, policy_cpu; - struct cpufreq_freqs freqs; - - if (cpufreq_frequency_table_target(policy, &speedstep_freqs[0], - target_freq, relation, &newstate)) - return -EINVAL; + unsigned int policy_cpu; policy_cpu = cpumask_any_and(policy->cpus, cpu_online_mask); - freqs.old = speedstep_get(policy_cpu); - freqs.new = speedstep_freqs[newstate].frequency; - - pr_debug("transiting from %u to %u kHz\n", freqs.old, freqs.new); - /* no transition necessary */ - if (freqs.old == freqs.new) - return 0; - - cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE); - - smp_call_function_single(policy_cpu, _speedstep_set_state, &newstate, + smp_call_function_single(policy_cpu, _speedstep_set_state, &index, true); - cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); - return 0; } -/** - * speedstep_verify - verifies a new CPUFreq policy - * @policy: new policy - * - * Limit must be within speedstep_low_freq and speedstep_high_freq, with - * at least one border included. - */ -static int speedstep_verify(struct cpufreq_policy *policy) -{ - return cpufreq_frequency_table_verify(policy, &speedstep_freqs[0]); -} - struct get_freqs { struct cpufreq_policy *policy; int ret; @@ -320,13 +288,12 @@ static void get_freqs_on_cpu(void *_get_freqs) static int speedstep_cpu_init(struct cpufreq_policy *policy) { - int result; - unsigned int policy_cpu, speed; + unsigned int policy_cpu; struct get_freqs gf; /* only run on CPU to be set, or on its sibling */ #ifdef CONFIG_SMP - cpumask_copy(policy->cpus, cpu_sibling_mask(policy->cpu)); + cpumask_copy(policy->cpus, topology_sibling_cpumask(policy->cpu)); #endif policy_cpu = cpumask_any_and(policy->cpus, cpu_online_mask); @@ -336,62 +303,26 @@ static int speedstep_cpu_init(struct cpufreq_policy *policy) if (gf.ret) return gf.ret; - /* get current speed setting */ - speed = speedstep_get(policy_cpu); - if (!speed) - return -EIO; - - pr_debug("currently at %s speed setting - %i MHz\n", - (speed == speedstep_freqs[SPEEDSTEP_LOW].frequency) - ? "low" : "high", - (speed / 1000)); - - /* cpuinfo and default policy values */ - policy->cur = speed; - - result = cpufreq_frequency_table_cpuinfo(policy, speedstep_freqs); - if (result) - return result; - - cpufreq_frequency_table_get_attr(speedstep_freqs, policy->cpu); + policy->freq_table = speedstep_freqs; return 0; } -static int speedstep_cpu_exit(struct cpufreq_policy *policy) -{ - cpufreq_frequency_table_put_attr(policy->cpu); - return 0; -} - -static struct freq_attr *speedstep_attr[] = { - &cpufreq_freq_attr_scaling_available_freqs, - NULL, -}; - - static struct cpufreq_driver speedstep_driver = { .name = "speedstep-ich", - .verify = speedstep_verify, - .target = speedstep_target, + .verify = cpufreq_generic_frequency_table_verify, + .target_index = speedstep_target, .init = speedstep_cpu_init, - .exit = speedstep_cpu_exit, .get = speedstep_get, - .owner = THIS_MODULE, - .attr = speedstep_attr, }; static const struct x86_cpu_id ss_smi_ids[] = { - { X86_VENDOR_INTEL, 6, 0xb, }, - { X86_VENDOR_INTEL, 6, 0x8, }, - { X86_VENDOR_INTEL, 15, 2 }, + X86_MATCH_VENDOR_FAM_MODEL(INTEL, 6, 0x8, 0), + X86_MATCH_VENDOR_FAM_MODEL(INTEL, 6, 0xb, 0), + X86_MATCH_VENDOR_FAM_MODEL(INTEL, 15, 0x2, 0), {} }; -#if 0 -/* Autoload or not? Do not for now. */ -MODULE_DEVICE_TABLE(x86cpu, ss_smi_ids); -#endif /** * speedstep_init - initializes the SpeedStep CPUFreq driver @@ -445,8 +376,7 @@ static void __exit speedstep_exit(void) } -MODULE_AUTHOR("Dave Jones <davej@redhat.com>, " - "Dominik Brodowski <linux@brodo.de>"); +MODULE_AUTHOR("Dave Jones, Dominik Brodowski <linux@brodo.de>"); MODULE_DESCRIPTION("Speedstep driver for Intel mobile processors on chipsets " "with ICH-M southbridges."); MODULE_LICENSE("GPL"); diff --git a/drivers/cpufreq/speedstep-lib.c b/drivers/cpufreq/speedstep-lib.c index 7047821a7f8a..f8b42e981635 100644 --- a/drivers/cpufreq/speedstep-lib.c +++ b/drivers/cpufreq/speedstep-lib.c @@ -1,13 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de> * - * Licensed under the terms of the GNU GPL License version 2. - * * Library for common functions for Intel SpeedStep v.1 and v.2 support * * BIG FAT DISCLAIMER: Work in progress code. Possibly *dangerous* */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + #include <linux/kernel.h> #include <linux/module.h> #include <linux/moduleparam.h> @@ -33,7 +34,7 @@ static int relaxed_check; static unsigned int pentium3_get_frequency(enum speedstep_processor processor) { /* See table 14 of p3_ds.pdf and table 22 of 29834003.pdf */ - struct { + static const struct { unsigned int ratio; /* Frequency Multiplier (x10) */ u8 bitmap; /* power on configuration bits [27, 25:22] (in MSR 0x2a) */ @@ -56,7 +57,7 @@ static unsigned int pentium3_get_frequency(enum speedstep_processor processor) }; /* PIII(-M) FSB settings: see table b1-b of 24547206.pdf */ - struct { + static const struct { unsigned int value; /* Front Side Bus speed in MHz */ u8 bitmap; /* power on configuration bits [18: 19] (in MSR 0x2a) */ @@ -153,7 +154,7 @@ static unsigned int pentium_core_get_frequency(void) fsb = 333333; break; default: - printk(KERN_ERR "PCORE - MSR_FSB_FREQ undefined value"); + pr_err("PCORE - MSR_FSB_FREQ undefined value\n"); } rdmsr(MSR_IA32_EBL_CR_POWERON, msr_lo, msr_tmp); @@ -239,7 +240,7 @@ unsigned int speedstep_get_frequency(enum speedstep_processor processor) return pentium3_get_frequency(processor); default: return 0; - }; + } return 0; } EXPORT_SYMBOL_GPL(speedstep_get_frequency); @@ -250,7 +251,7 @@ EXPORT_SYMBOL_GPL(speedstep_get_frequency); *********************************************************************/ /* Keep in sync with the x86_cpu_id tables in the different modules */ -unsigned int speedstep_detect_processor(void) +enum speedstep_processor speedstep_detect_processor(void) { struct cpuinfo_x86 *c = &cpu_data(0); u32 ebx, msr_lo, msr_hi; @@ -270,9 +271,9 @@ unsigned int speedstep_detect_processor(void) ebx = cpuid_ebx(0x00000001); ebx &= 0x000000FF; - pr_debug("ebx value is %x, x86_mask is %x\n", ebx, c->x86_mask); + pr_debug("ebx value is %x, x86_stepping is %x\n", ebx, c->x86_stepping); - switch (c->x86_mask) { + switch (c->x86_stepping) { case 4: /* * B-stepping [M-P4-M] @@ -359,13 +360,13 @@ unsigned int speedstep_detect_processor(void) msr_lo, msr_hi); if ((msr_hi & (1<<18)) && (relaxed_check ? 1 : (msr_hi & (3<<24)))) { - if (c->x86_mask == 0x01) { + if (c->x86_stepping == 0x01) { pr_debug("early PIII version\n"); return SPEEDSTEP_CPU_PIII_C_EARLY; } else return SPEEDSTEP_CPU_PIII_C; } - + fallthrough; default: return 0; } @@ -377,16 +378,16 @@ EXPORT_SYMBOL_GPL(speedstep_detect_processor); * DETECT SPEEDSTEP SPEEDS * *********************************************************************/ -unsigned int speedstep_get_freqs(enum speedstep_processor processor, - unsigned int *low_speed, - unsigned int *high_speed, - unsigned int *transition_latency, - void (*set_state) (unsigned int state)) +int speedstep_get_freqs(enum speedstep_processor processor, + unsigned int *low_speed, + unsigned int *high_speed, + unsigned int *transition_latency, + void (*set_state)(unsigned int state)) { unsigned int prev_speed; - unsigned int ret = 0; unsigned long flags; - struct timeval tv1, tv2; + ktime_t tv1, tv2; + int ret = 0; if ((!processor) || (!low_speed) || (!high_speed) || (!set_state)) return -EINVAL; @@ -400,6 +401,7 @@ unsigned int speedstep_get_freqs(enum speedstep_processor processor, pr_debug("previous speed is %u\n", prev_speed); + preempt_disable(); local_irq_save(flags); /* switch to low state */ @@ -414,14 +416,14 @@ unsigned int speedstep_get_freqs(enum speedstep_processor processor, /* start latency measurement */ if (transition_latency) - do_gettimeofday(&tv1); + tv1 = ktime_get(); /* switch to high state */ set_state(SPEEDSTEP_HIGH); /* end latency measurement */ if (transition_latency) - do_gettimeofday(&tv2); + tv2 = ktime_get(); *high_speed = speedstep_get_frequency(processor); if (!*high_speed) { @@ -441,8 +443,7 @@ unsigned int speedstep_get_freqs(enum speedstep_processor processor, set_state(SPEEDSTEP_LOW); if (transition_latency) { - *transition_latency = (tv2.tv_sec - tv1.tv_sec) * USEC_PER_SEC + - tv2.tv_usec - tv1.tv_usec; + *transition_latency = ktime_to_us(ktime_sub(tv2, tv1)); pr_debug("transition latency is %u uSec\n", *transition_latency); /* convert uSec to nSec and add 20% for safety reasons */ @@ -453,17 +454,16 @@ unsigned int speedstep_get_freqs(enum speedstep_processor processor, */ if (*transition_latency > 10000000 || *transition_latency < 50000) { - printk(KERN_WARNING PFX "frequency transition " - "measured seems out of range (%u " - "nSec), falling back to a safe one of" - "%u nSec.\n", - *transition_latency, 500000); + pr_warn("frequency transition measured seems out of range (%u nSec), falling back to a safe one of %u nSec\n", + *transition_latency, 500000); *transition_latency = 500000; } } out: local_irq_restore(flags); + preempt_enable(); + return ret; } EXPORT_SYMBOL_GPL(speedstep_get_freqs); diff --git a/drivers/cpufreq/speedstep-lib.h b/drivers/cpufreq/speedstep-lib.h index 70d9cea1219d..48329647d4c4 100644 --- a/drivers/cpufreq/speedstep-lib.h +++ b/drivers/cpufreq/speedstep-lib.h @@ -1,8 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de> * - * Licensed under the terms of the GNU GPL License version 2. - * * Library for common functions for Intel SpeedStep v.1 and v.2 support * * BIG FAT DISCLAIMER: Work in progress code. Possibly *dangerous* @@ -42,8 +41,8 @@ extern unsigned int speedstep_get_frequency(enum speedstep_processor processor); * SPEEDSTEP_LOW; the second argument is zero so that no * cpufreq_notify_transition calls are initiated. */ -extern unsigned int speedstep_get_freqs(enum speedstep_processor processor, - unsigned int *low_speed, - unsigned int *high_speed, - unsigned int *transition_latency, - void (*set_state) (unsigned int state)); +extern int speedstep_get_freqs(enum speedstep_processor processor, + unsigned int *low_speed, + unsigned int *high_speed, + unsigned int *transition_latency, + void (*set_state)(unsigned int state)); diff --git a/drivers/cpufreq/speedstep-smi.c b/drivers/cpufreq/speedstep-smi.c index f5a6b70ee6c0..39265884c3f1 100644 --- a/drivers/cpufreq/speedstep-smi.c +++ b/drivers/cpufreq/speedstep-smi.c @@ -1,10 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Intel SpeedStep SMI driver. * * (C) 2003 Hiroshi Miura <miura@da-cha.org> - * - * Licensed under the terms of the GNU GPL License version 2. - * */ @@ -12,6 +10,8 @@ * SPEEDSTEP - DEFINITIONS * *********************************************************************/ +#define pr_fmt(fmt) "cpufreq: " fmt + #include <linux/kernel.h> #include <linux/module.h> #include <linux/moduleparam.h> @@ -42,9 +42,9 @@ static enum speedstep_processor speedstep_processor; * are in kHz for the time being. */ static struct cpufreq_frequency_table speedstep_freqs[] = { - {SPEEDSTEP_HIGH, 0}, - {SPEEDSTEP_LOW, 0}, - {0, CPUFREQ_TABLE_END}, + {0, SPEEDSTEP_HIGH, 0}, + {0, SPEEDSTEP_LOW, 0}, + {0, 0, CPUFREQ_TABLE_END}, }; #define GET_SPEEDSTEP_OWNER 0 @@ -141,38 +141,6 @@ static int speedstep_smi_get_freqs(unsigned int *low, unsigned int *high) } /** - * speedstep_get_state - set the SpeedStep state - * @state: processor frequency state (SPEEDSTEP_LOW or SPEEDSTEP_HIGH) - * - */ -static int speedstep_get_state(void) -{ - u32 function = GET_SPEEDSTEP_STATE; - u32 result, state, edi, command, dummy; - - command = (smi_sig & 0xffffff00) | (smi_cmd & 0xff); - - pr_debug("trying to determine current setting with command %x " - "at port %x\n", command, smi_port); - - __asm__ __volatile__( - "push %%ebp\n" - "out %%al, (%%dx)\n" - "pop %%ebp\n" - : "=a" (result), - "=b" (state), "=D" (edi), - "=c" (dummy), "=d" (dummy), "=S" (dummy) - : "a" (command), "b" (function), "c" (0), - "d" (smi_port), "S" (0), "D" (0) - ); - - pr_debug("state is %x, result is %x\n", state, result); - - return state & 1; -} - - -/** * speedstep_set_state - set the SpeedStep state * @state: new processor frequency state (SPEEDSTEP_LOW or SPEEDSTEP_HIGH) * @@ -188,6 +156,7 @@ static void speedstep_set_state(unsigned int state) return; /* Disable IRQs */ + preempt_disable(); local_irq_save(flags); command = (smi_sig & 0xffffff00) | (smi_cmd & 0xff); @@ -198,9 +167,19 @@ static void speedstep_set_state(unsigned int state) do { if (retry) { + /* + * We need to enable interrupts, otherwise the blockage + * won't resolve. + * + * We disable preemption so that other processes don't + * run. If other processes were running, they could + * submit more DMA requests, making the blockage worse. + */ pr_debug("retry %u, previous result %u, waiting...\n", retry, result); + local_irq_enable(); mdelay(retry * 50); + local_irq_disable(); } retry++; __asm__ __volatile__( @@ -217,6 +196,7 @@ static void speedstep_set_state(unsigned int state) /* enable IRQs */ local_irq_restore(flags); + preempt_enable(); if (new_state == state) pr_debug("change to %u MHz succeeded after %u tries " @@ -224,9 +204,8 @@ static void speedstep_set_state(unsigned int state) (speedstep_freqs[new_state].frequency / 1000), retry, result); else - printk(KERN_ERR "cpufreq: change to state %u " - "failed with new_state %u and result %u\n", - state, new_state, result); + pr_err("change to state %u failed with new_state %u and result %u\n", + state, new_state, result); return; } @@ -235,52 +214,21 @@ static void speedstep_set_state(unsigned int state) /** * speedstep_target - set a new CPUFreq policy * @policy: new policy - * @target_freq: new freq - * @relation: + * @index: index of new freq * * Sets a new CPUFreq policy/freq. */ -static int speedstep_target(struct cpufreq_policy *policy, - unsigned int target_freq, unsigned int relation) +static int speedstep_target(struct cpufreq_policy *policy, unsigned int index) { - unsigned int newstate = 0; - struct cpufreq_freqs freqs; - - if (cpufreq_frequency_table_target(policy, &speedstep_freqs[0], - target_freq, relation, &newstate)) - return -EINVAL; - - freqs.old = speedstep_freqs[speedstep_get_state()].frequency; - freqs.new = speedstep_freqs[newstate].frequency; - - if (freqs.old == freqs.new) - return 0; - - cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE); - speedstep_set_state(newstate); - cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); + speedstep_set_state(index); return 0; } -/** - * speedstep_verify - verifies a new CPUFreq policy - * @policy: new policy - * - * Limit must be within speedstep_low_freq and speedstep_high_freq, with - * at least one border included. - */ -static int speedstep_verify(struct cpufreq_policy *policy) -{ - return cpufreq_frequency_table_verify(policy, &speedstep_freqs[0]); -} - - static int speedstep_cpu_init(struct cpufreq_policy *policy) { int result; - unsigned int speed, state; unsigned int *low, *high; /* capability check */ @@ -316,34 +264,11 @@ static int speedstep_cpu_init(struct cpufreq_policy *policy) pr_debug("workaround worked.\n"); } - /* get current speed setting */ - state = speedstep_get_state(); - speed = speedstep_freqs[state].frequency; - - pr_debug("currently at %s speed setting - %i MHz\n", - (speed == speedstep_freqs[SPEEDSTEP_LOW].frequency) - ? "low" : "high", - (speed / 1000)); - - /* cpuinfo and default policy values */ - policy->cpuinfo.transition_latency = CPUFREQ_ETERNAL; - policy->cur = speed; - - result = cpufreq_frequency_table_cpuinfo(policy, speedstep_freqs); - if (result) - return result; - - cpufreq_frequency_table_get_attr(speedstep_freqs, policy->cpu); + policy->freq_table = speedstep_freqs; return 0; } -static int speedstep_cpu_exit(struct cpufreq_policy *policy) -{ - cpufreq_frequency_table_put_attr(policy->cpu); - return 0; -} - static unsigned int speedstep_get(unsigned int cpu) { if (cpu) @@ -362,33 +287,22 @@ static int speedstep_resume(struct cpufreq_policy *policy) return result; } -static struct freq_attr *speedstep_attr[] = { - &cpufreq_freq_attr_scaling_available_freqs, - NULL, -}; - static struct cpufreq_driver speedstep_driver = { .name = "speedstep-smi", - .verify = speedstep_verify, - .target = speedstep_target, + .flags = CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING, + .verify = cpufreq_generic_frequency_table_verify, + .target_index = speedstep_target, .init = speedstep_cpu_init, - .exit = speedstep_cpu_exit, .get = speedstep_get, .resume = speedstep_resume, - .owner = THIS_MODULE, - .attr = speedstep_attr, }; static const struct x86_cpu_id ss_smi_ids[] = { - { X86_VENDOR_INTEL, 6, 0xb, }, - { X86_VENDOR_INTEL, 6, 0x8, }, - { X86_VENDOR_INTEL, 15, 2 }, + X86_MATCH_VENDOR_FAM_MODEL(INTEL, 6, 0x8, 0), + X86_MATCH_VENDOR_FAM_MODEL(INTEL, 6, 0xb, 0), + X86_MATCH_VENDOR_FAM_MODEL(INTEL, 15, 0x2, 0), {} }; -#if 0 -/* Not auto loaded currently */ -MODULE_DEVICE_TABLE(x86cpu, ss_smi_ids); -#endif /** * speedstep_init - initializes the SpeedStep CPUFreq driver @@ -418,8 +332,8 @@ static int __init speedstep_init(void) return -ENODEV; } - pr_debug("signature:0x%.8ulx, command:0x%.8ulx, " - "event:0x%.8ulx, perf_level:0x%.8ulx.\n", + pr_debug("signature:0x%.8x, command:0x%.8x, " + "event:0x%.8x, perf_level:0x%.8x.\n", ist_info.signature, ist_info.command, ist_info.event, ist_info.perf_level); @@ -459,7 +373,7 @@ static void __exit speedstep_exit(void) cpufreq_unregister_driver(&speedstep_driver); } -module_param(smi_port, int, 0444); +module_param_hw(smi_port, int, ioport, 0444); module_param(smi_cmd, int, 0444); module_param(smi_sig, uint, 0444); diff --git a/drivers/cpufreq/sti-cpufreq.c b/drivers/cpufreq/sti-cpufreq.c new file mode 100644 index 000000000000..b15b3142b5fe --- /dev/null +++ b/drivers/cpufreq/sti-cpufreq.c @@ -0,0 +1,304 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Match running platform with pre-defined OPP values for CPUFreq + * + * Author: Ajit Pal Singh <ajitpal.singh@st.com> + * Lee Jones <lee.jones@linaro.org> + * + * Copyright (C) 2015 STMicroelectronics (R&D) Limited + */ + +#include <linux/cpu.h> +#include <linux/io.h> +#include <linux/mfd/syscon.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/platform_device.h> +#include <linux/pm_opp.h> +#include <linux/regmap.h> + +#define VERSION_ELEMENTS 3 +#define MAX_PCODE_NAME_LEN 16 + +#define VERSION_SHIFT 28 +#define HW_INFO_INDEX 1 +#define MAJOR_ID_INDEX 1 +#define MINOR_ID_INDEX 2 + +/* + * Only match on "suitable for ALL versions" entries + * + * This will be used with the BIT() macro. It sets the + * top bit of a 32bit value and is equal to 0x80000000. + */ +#define DEFAULT_VERSION 31 + +enum { + PCODE = 0, + SUBSTRATE, + DVFS_MAX_REGFIELDS, +}; + +/** + * struct sti_cpufreq_ddata - ST CPUFreq Driver Data + * + * @cpu: CPU's OF node + * @syscfg_eng: Engineering Syscon register map + * @syscfg: Syscon register map + */ +static struct sti_cpufreq_ddata { + struct device *cpu; + struct regmap *syscfg_eng; + struct regmap *syscfg; +} ddata; + +static int sti_cpufreq_fetch_major(void) { + struct device_node *np = ddata.cpu->of_node; + struct device *dev = ddata.cpu; + unsigned int major_offset; + unsigned int socid; + int ret; + + ret = of_property_read_u32_index(np, "st,syscfg", + MAJOR_ID_INDEX, &major_offset); + if (ret) { + dev_err(dev, "No major number offset provided in %pOF [%d]\n", + np, ret); + return ret; + } + + ret = regmap_read(ddata.syscfg, major_offset, &socid); + if (ret) { + dev_err(dev, "Failed to read major number from syscon [%d]\n", + ret); + return ret; + } + + return ((socid >> VERSION_SHIFT) & 0xf) + 1; +} + +static int sti_cpufreq_fetch_minor(void) +{ + struct device *dev = ddata.cpu; + struct device_node *np = dev->of_node; + unsigned int minor_offset; + unsigned int minid; + int ret; + + ret = of_property_read_u32_index(np, "st,syscfg-eng", + MINOR_ID_INDEX, &minor_offset); + if (ret) { + dev_err(dev, + "No minor number offset provided %pOF [%d]\n", + np, ret); + return ret; + } + + ret = regmap_read(ddata.syscfg_eng, minor_offset, &minid); + if (ret) { + dev_err(dev, + "Failed to read the minor number from syscon [%d]\n", + ret); + return ret; + } + + return minid & 0xf; +} + +static int sti_cpufreq_fetch_regmap_field(const struct reg_field *reg_fields, + int hw_info_offset, int field) +{ + struct regmap_field *regmap_field; + struct reg_field reg_field = reg_fields[field]; + struct device *dev = ddata.cpu; + unsigned int value; + int ret; + + reg_field.reg = hw_info_offset; + regmap_field = devm_regmap_field_alloc(dev, + ddata.syscfg_eng, + reg_field); + if (IS_ERR(regmap_field)) { + dev_err(dev, "Failed to allocate reg field\n"); + return PTR_ERR(regmap_field); + } + + ret = regmap_field_read(regmap_field, &value); + if (ret) { + dev_err(dev, "Failed to read %s code\n", + field ? "SUBSTRATE" : "PCODE"); + return ret; + } + + return value; +} + +static const struct reg_field sti_stih407_dvfs_regfields[DVFS_MAX_REGFIELDS] = { + [PCODE] = REG_FIELD(0, 16, 19), + [SUBSTRATE] = REG_FIELD(0, 0, 2), +}; + +static const struct reg_field *sti_cpufreq_match(void) +{ + if (of_machine_is_compatible("st,stih407") || + of_machine_is_compatible("st,stih410") || + of_machine_is_compatible("st,stih418")) + return sti_stih407_dvfs_regfields; + + return NULL; +} + +static int sti_cpufreq_set_opp_info(void) +{ + struct device *dev = ddata.cpu; + struct device_node *np = dev->of_node; + const struct reg_field *reg_fields; + unsigned int hw_info_offset; + unsigned int version[VERSION_ELEMENTS]; + int pcode, substrate, major, minor; + int opp_token, ret; + char name[MAX_PCODE_NAME_LEN]; + struct dev_pm_opp_config config = { + .supported_hw = version, + .supported_hw_count = ARRAY_SIZE(version), + .prop_name = name, + }; + + reg_fields = sti_cpufreq_match(); + if (!reg_fields) { + dev_err(dev, "This SoC doesn't support voltage scaling\n"); + return -ENODEV; + } + + ret = of_property_read_u32_index(np, "st,syscfg-eng", + HW_INFO_INDEX, &hw_info_offset); + if (ret) { + dev_warn(dev, "Failed to read HW info offset from DT\n"); + substrate = DEFAULT_VERSION; + pcode = 0; + goto use_defaults; + } + + pcode = sti_cpufreq_fetch_regmap_field(reg_fields, + hw_info_offset, + PCODE); + if (pcode < 0) { + dev_warn(dev, "Failed to obtain process code\n"); + /* Use default pcode */ + pcode = 0; + } + + substrate = sti_cpufreq_fetch_regmap_field(reg_fields, + hw_info_offset, + SUBSTRATE); + if (substrate) { + dev_warn(dev, "Failed to obtain substrate code\n"); + /* Use default substrate */ + substrate = DEFAULT_VERSION; + } + +use_defaults: + major = sti_cpufreq_fetch_major(); + if (major < 0) { + dev_err(dev, "Failed to obtain major version\n"); + /* Use default major number */ + major = DEFAULT_VERSION; + } + + minor = sti_cpufreq_fetch_minor(); + if (minor < 0) { + dev_err(dev, "Failed to obtain minor version\n"); + /* Use default minor number */ + minor = DEFAULT_VERSION; + } + + snprintf(name, MAX_PCODE_NAME_LEN, "pcode%d", pcode); + + version[0] = BIT(major); + version[1] = BIT(minor); + version[2] = BIT(substrate); + + opp_token = dev_pm_opp_set_config(dev, &config); + if (opp_token < 0) { + dev_err(dev, "Failed to set OPP config\n"); + return opp_token; + } + + dev_dbg(dev, "pcode: %d major: %d minor: %d substrate: %d\n", + pcode, major, minor, substrate); + dev_dbg(dev, "version[0]: %x version[1]: %x version[2]: %x\n", + version[0], version[1], version[2]); + + return 0; +} + +static int sti_cpufreq_fetch_syscon_registers(void) +{ + struct device *dev = ddata.cpu; + struct device_node *np = dev->of_node; + + ddata.syscfg = syscon_regmap_lookup_by_phandle(np, "st,syscfg"); + if (IS_ERR(ddata.syscfg)) { + dev_err(dev, "\"st,syscfg\" not supplied\n"); + return PTR_ERR(ddata.syscfg); + } + + ddata.syscfg_eng = syscon_regmap_lookup_by_phandle(np, "st,syscfg-eng"); + if (IS_ERR(ddata.syscfg_eng)) { + dev_err(dev, "\"st,syscfg-eng\" not supplied\n"); + return PTR_ERR(ddata.syscfg_eng); + } + + return 0; +} + +static int __init sti_cpufreq_init(void) +{ + int ret; + + if ((!of_machine_is_compatible("st,stih407")) && + (!of_machine_is_compatible("st,stih410")) && + (!of_machine_is_compatible("st,stih418"))) + return -ENODEV; + + ddata.cpu = get_cpu_device(0); + if (!ddata.cpu) { + dev_err(ddata.cpu, "Failed to get device for CPU0\n"); + goto skip_voltage_scaling; + } + + if (!of_property_present(ddata.cpu->of_node, "operating-points-v2")) { + dev_err(ddata.cpu, "OPP-v2 not supported\n"); + goto skip_voltage_scaling; + } + + ret = sti_cpufreq_fetch_syscon_registers(); + if (ret) + goto skip_voltage_scaling; + + ret = sti_cpufreq_set_opp_info(); + if (!ret) + goto register_cpufreq_dt; + +skip_voltage_scaling: + dev_err(ddata.cpu, "Not doing voltage scaling\n"); + +register_cpufreq_dt: + platform_device_register_simple("cpufreq-dt", -1, NULL, 0); + + return 0; +} +module_init(sti_cpufreq_init); + +static const struct of_device_id __maybe_unused sti_cpufreq_of_match[] = { + { .compatible = "st,stih407" }, + { .compatible = "st,stih410" }, + { .compatible = "st,stih418" }, + { }, +}; +MODULE_DEVICE_TABLE(of, sti_cpufreq_of_match); + +MODULE_DESCRIPTION("STMicroelectronics CPUFreq/OPP driver"); +MODULE_AUTHOR("Ajitpal Singh <ajitpal.singh@st.com>"); +MODULE_AUTHOR("Lee Jones <lee.jones@linaro.org>"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/cpufreq/sun50i-cpufreq-nvmem.c b/drivers/cpufreq/sun50i-cpufreq-nvmem.c new file mode 100644 index 000000000000..4fffc8e83692 --- /dev/null +++ b/drivers/cpufreq/sun50i-cpufreq-nvmem.c @@ -0,0 +1,371 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Allwinner CPUFreq nvmem based driver + * + * The sun50i-cpufreq-nvmem driver reads the efuse value from the SoC to + * provide the OPP framework with required information. + * + * Copyright (C) 2019 Yangtao Li <tiny.windzz@gmail.com> + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/arm-smccc.h> +#include <linux/cpu.h> +#include <linux/module.h> +#include <linux/nvmem-consumer.h> +#include <linux/of.h> +#include <linux/platform_device.h> +#include <linux/pm_opp.h> +#include <linux/slab.h> + +#define NVMEM_MASK 0x7 +#define NVMEM_SHIFT 5 + +#define SUN50I_A100_NVMEM_MASK 0xf +#define SUN50I_A100_NVMEM_SHIFT 12 + +static struct platform_device *cpufreq_dt_pdev, *sun50i_cpufreq_pdev; + +struct sunxi_cpufreq_data { + u32 (*efuse_xlate)(u32 speedbin); +}; + +static u32 sun50i_h6_efuse_xlate(u32 speedbin) +{ + u32 efuse_value; + + efuse_value = (speedbin >> NVMEM_SHIFT) & NVMEM_MASK; + + /* + * We treat unexpected efuse values as if the SoC was from + * the slowest bin. Expected efuse values are 1-3, slowest + * to fastest. + */ + if (efuse_value >= 1 && efuse_value <= 3) + return efuse_value - 1; + else + return 0; +} + +static u32 sun50i_a100_efuse_xlate(u32 speedbin) +{ + u32 efuse_value; + + efuse_value = (speedbin >> SUN50I_A100_NVMEM_SHIFT) & + SUN50I_A100_NVMEM_MASK; + + switch (efuse_value) { + case 0b100: + return 2; + case 0b010: + return 1; + default: + return 0; + } +} + +static int get_soc_id_revision(void) +{ +#ifdef CONFIG_HAVE_ARM_SMCCC_DISCOVERY + return arm_smccc_get_soc_id_revision(); +#else + return SMCCC_RET_NOT_SUPPORTED; +#endif +} + +/* + * Judging by the OPP tables in the vendor BSP, the quality order of the + * returned speedbin index is 4 -> 0/2 -> 3 -> 1, from worst to best. + * 0 and 2 seem identical from the OPP tables' point of view. + */ +static u32 sun50i_h616_efuse_xlate(u32 speedbin) +{ + int ver_bits = get_soc_id_revision(); + u32 value = 0; + + switch (speedbin & 0xffff) { + case 0x2000: + value = 0; + break; + case 0x2400: + case 0x7400: + case 0x2c00: + case 0x7c00: + if (ver_bits != SMCCC_RET_NOT_SUPPORTED && ver_bits <= 1) { + /* ic version A/B */ + value = 1; + } else { + /* ic version C and later version */ + value = 2; + } + break; + case 0x5000: + case 0x5400: + case 0x6000: + value = 3; + break; + case 0x5c00: + value = 4; + break; + case 0x5d00: + value = 0; + break; + case 0x6c00: + value = 5; + break; + default: + pr_warn("sun50i-cpufreq-nvmem: unknown speed bin 0x%x, using default bin 0\n", + speedbin & 0xffff); + value = 0; + break; + } + + return value; +} + +static struct sunxi_cpufreq_data sun50i_h6_cpufreq_data = { + .efuse_xlate = sun50i_h6_efuse_xlate, +}; + +static struct sunxi_cpufreq_data sun50i_a100_cpufreq_data = { + .efuse_xlate = sun50i_a100_efuse_xlate, +}; + +static struct sunxi_cpufreq_data sun50i_h616_cpufreq_data = { + .efuse_xlate = sun50i_h616_efuse_xlate, +}; + +static const struct of_device_id cpu_opp_match_list[] = { + { .compatible = "allwinner,sun50i-h6-operating-points", + .data = &sun50i_h6_cpufreq_data, + }, + { .compatible = "allwinner,sun50i-a100-operating-points", + .data = &sun50i_a100_cpufreq_data, + }, + { .compatible = "allwinner,sun50i-h616-operating-points", + .data = &sun50i_h616_cpufreq_data, + }, + {} +}; + +/** + * dt_has_supported_hw() - Check if any OPPs use opp-supported-hw + * + * If we ask the cpufreq framework to use the opp-supported-hw feature, it + * will ignore every OPP node without that DT property. If none of the OPPs + * have it, the driver will fail probing, due to the lack of OPPs. + * + * Returns true if we have at least one OPP with the opp-supported-hw property. + */ +static bool dt_has_supported_hw(void) +{ + bool has_opp_supported_hw = false; + struct device *cpu_dev; + + cpu_dev = get_cpu_device(0); + if (!cpu_dev) + return false; + + struct device_node *np __free(device_node) = + dev_pm_opp_of_get_opp_desc_node(cpu_dev); + if (!np) + return false; + + for_each_child_of_node_scoped(np, opp) { + if (of_property_present(opp, "opp-supported-hw")) { + has_opp_supported_hw = true; + break; + } + } + + return has_opp_supported_hw; +} + +/** + * sun50i_cpufreq_get_efuse() - Determine speed grade from efuse value + * + * Returns non-negative speed bin index on success, a negative error + * value otherwise. + */ +static int sun50i_cpufreq_get_efuse(void) +{ + const struct sunxi_cpufreq_data *opp_data; + struct nvmem_cell *speedbin_nvmem; + const struct of_device_id *match; + struct device *cpu_dev; + void *speedbin_ptr; + u32 speedbin = 0; + size_t len; + int ret; + + cpu_dev = get_cpu_device(0); + if (!cpu_dev) + return -ENODEV; + + struct device_node *np __free(device_node) = + dev_pm_opp_of_get_opp_desc_node(cpu_dev); + if (!np) + return -ENOENT; + + match = of_match_node(cpu_opp_match_list, np); + if (!match) + return -ENOENT; + + opp_data = match->data; + + speedbin_nvmem = of_nvmem_cell_get(np, NULL); + if (IS_ERR(speedbin_nvmem)) + return dev_err_probe(cpu_dev, PTR_ERR(speedbin_nvmem), + "Could not get nvmem cell\n"); + + speedbin_ptr = nvmem_cell_read(speedbin_nvmem, &len); + nvmem_cell_put(speedbin_nvmem); + if (IS_ERR(speedbin_ptr)) + return PTR_ERR(speedbin_ptr); + + if (len <= 4) + memcpy(&speedbin, speedbin_ptr, len); + speedbin = le32_to_cpu(speedbin); + + ret = opp_data->efuse_xlate(speedbin); + + kfree(speedbin_ptr); + + return ret; +}; + +static int sun50i_cpufreq_nvmem_probe(struct platform_device *pdev) +{ + int *opp_tokens; + char name[] = "speedXXXXXXXXXXX"; /* Integers can take 11 chars max */ + unsigned int cpu, supported_hw; + struct dev_pm_opp_config config = {}; + int speed; + int ret; + + opp_tokens = kcalloc(num_possible_cpus(), sizeof(*opp_tokens), + GFP_KERNEL); + if (!opp_tokens) + return -ENOMEM; + + speed = sun50i_cpufreq_get_efuse(); + if (speed < 0) { + kfree(opp_tokens); + return speed; + } + + /* + * We need at least one OPP with the "opp-supported-hw" property, + * or else the upper layers will ignore every OPP and will bail out. + */ + if (dt_has_supported_hw()) { + supported_hw = 1U << speed; + config.supported_hw = &supported_hw; + config.supported_hw_count = 1; + } + + snprintf(name, sizeof(name), "speed%d", speed); + config.prop_name = name; + + for_each_present_cpu(cpu) { + struct device *cpu_dev = get_cpu_device(cpu); + + if (!cpu_dev) { + ret = -ENODEV; + goto free_opp; + } + + ret = dev_pm_opp_set_config(cpu_dev, &config); + if (ret < 0) + goto free_opp; + + opp_tokens[cpu] = ret; + } + + cpufreq_dt_pdev = platform_device_register_simple("cpufreq-dt", -1, + NULL, 0); + if (!IS_ERR(cpufreq_dt_pdev)) { + platform_set_drvdata(pdev, opp_tokens); + return 0; + } + + ret = PTR_ERR(cpufreq_dt_pdev); + pr_err("Failed to register platform device\n"); + +free_opp: + for_each_present_cpu(cpu) + dev_pm_opp_clear_config(opp_tokens[cpu]); + kfree(opp_tokens); + + return ret; +} + +static void sun50i_cpufreq_nvmem_remove(struct platform_device *pdev) +{ + int *opp_tokens = platform_get_drvdata(pdev); + unsigned int cpu; + + platform_device_unregister(cpufreq_dt_pdev); + + for_each_present_cpu(cpu) + dev_pm_opp_clear_config(opp_tokens[cpu]); + + kfree(opp_tokens); +} + +static struct platform_driver sun50i_cpufreq_driver = { + .probe = sun50i_cpufreq_nvmem_probe, + .remove = sun50i_cpufreq_nvmem_remove, + .driver = { + .name = "sun50i-cpufreq-nvmem", + }, +}; + +static const struct of_device_id sun50i_cpufreq_match_list[] = { + { .compatible = "allwinner,sun50i-h6" }, + { .compatible = "allwinner,sun50i-a100" }, + { .compatible = "allwinner,sun50i-h616" }, + { .compatible = "allwinner,sun50i-h618" }, + { .compatible = "allwinner,sun50i-h700" }, + {} +}; +MODULE_DEVICE_TABLE(of, sun50i_cpufreq_match_list); + +/* + * Since the driver depends on nvmem drivers, which may return EPROBE_DEFER, + * all the real activity is done in the probe, which may be defered as well. + * The init here is only registering the driver and the platform device. + */ +static int __init sun50i_cpufreq_init(void) +{ + int ret; + + if (!of_machine_device_match(sun50i_cpufreq_match_list)) + return -ENODEV; + + ret = platform_driver_register(&sun50i_cpufreq_driver); + if (unlikely(ret < 0)) + return ret; + + sun50i_cpufreq_pdev = + platform_device_register_simple("sun50i-cpufreq-nvmem", + -1, NULL, 0); + ret = PTR_ERR_OR_ZERO(sun50i_cpufreq_pdev); + if (ret == 0) + return 0; + + platform_driver_unregister(&sun50i_cpufreq_driver); + return ret; +} +module_init(sun50i_cpufreq_init); + +static void __exit sun50i_cpufreq_exit(void) +{ + platform_device_unregister(sun50i_cpufreq_pdev); + platform_driver_unregister(&sun50i_cpufreq_driver); +} +module_exit(sun50i_cpufreq_exit); + +MODULE_DESCRIPTION("Sun50i-h6 cpufreq driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/cpufreq/tegra-cpufreq.c b/drivers/cpufreq/tegra-cpufreq.c deleted file mode 100644 index cd66b85d927c..000000000000 --- a/drivers/cpufreq/tegra-cpufreq.c +++ /dev/null @@ -1,291 +0,0 @@ -/* - * Copyright (C) 2010 Google, Inc. - * - * Author: - * Colin Cross <ccross@google.com> - * Based on arch/arm/plat-omap/cpu-omap.c, (C) 2005 Nokia Corporation - * - * This software is licensed under the terms of the GNU General Public - * License version 2, as published by the Free Software Foundation, and - * may be copied, distributed, and modified under those terms. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - */ - -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/types.h> -#include <linux/sched.h> -#include <linux/cpufreq.h> -#include <linux/delay.h> -#include <linux/init.h> -#include <linux/err.h> -#include <linux/clk.h> -#include <linux/io.h> -#include <linux/suspend.h> - -static struct cpufreq_frequency_table freq_table[] = { - { .frequency = 216000 }, - { .frequency = 312000 }, - { .frequency = 456000 }, - { .frequency = 608000 }, - { .frequency = 760000 }, - { .frequency = 816000 }, - { .frequency = 912000 }, - { .frequency = 1000000 }, - { .frequency = CPUFREQ_TABLE_END }, -}; - -#define NUM_CPUS 2 - -static struct clk *cpu_clk; -static struct clk *pll_x_clk; -static struct clk *pll_p_clk; -static struct clk *emc_clk; - -static unsigned long target_cpu_speed[NUM_CPUS]; -static DEFINE_MUTEX(tegra_cpu_lock); -static bool is_suspended; - -static int tegra_verify_speed(struct cpufreq_policy *policy) -{ - return cpufreq_frequency_table_verify(policy, freq_table); -} - -static unsigned int tegra_getspeed(unsigned int cpu) -{ - unsigned long rate; - - if (cpu >= NUM_CPUS) - return 0; - - rate = clk_get_rate(cpu_clk) / 1000; - return rate; -} - -static int tegra_cpu_clk_set_rate(unsigned long rate) -{ - int ret; - - /* - * Take an extra reference to the main pll so it doesn't turn - * off when we move the cpu off of it - */ - clk_prepare_enable(pll_x_clk); - - ret = clk_set_parent(cpu_clk, pll_p_clk); - if (ret) { - pr_err("Failed to switch cpu to clock pll_p\n"); - goto out; - } - - if (rate == clk_get_rate(pll_p_clk)) - goto out; - - ret = clk_set_rate(pll_x_clk, rate); - if (ret) { - pr_err("Failed to change pll_x to %lu\n", rate); - goto out; - } - - ret = clk_set_parent(cpu_clk, pll_x_clk); - if (ret) { - pr_err("Failed to switch cpu to clock pll_x\n"); - goto out; - } - -out: - clk_disable_unprepare(pll_x_clk); - return ret; -} - -static int tegra_update_cpu_speed(struct cpufreq_policy *policy, - unsigned long rate) -{ - int ret = 0; - struct cpufreq_freqs freqs; - - freqs.old = tegra_getspeed(0); - freqs.new = rate; - - if (freqs.old == freqs.new) - return ret; - - /* - * Vote on memory bus frequency based on cpu frequency - * This sets the minimum frequency, display or avp may request higher - */ - if (rate >= 816000) - clk_set_rate(emc_clk, 600000000); /* cpu 816 MHz, emc max */ - else if (rate >= 456000) - clk_set_rate(emc_clk, 300000000); /* cpu 456 MHz, emc 150Mhz */ - else - clk_set_rate(emc_clk, 100000000); /* emc 50Mhz */ - - cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE); - -#ifdef CONFIG_CPU_FREQ_DEBUG - printk(KERN_DEBUG "cpufreq-tegra: transition: %u --> %u\n", - freqs.old, freqs.new); -#endif - - ret = tegra_cpu_clk_set_rate(freqs.new * 1000); - if (ret) { - pr_err("cpu-tegra: Failed to set cpu frequency to %d kHz\n", - freqs.new); - freqs.new = freqs.old; - } - - cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); - - return ret; -} - -static unsigned long tegra_cpu_highest_speed(void) -{ - unsigned long rate = 0; - int i; - - for_each_online_cpu(i) - rate = max(rate, target_cpu_speed[i]); - return rate; -} - -static int tegra_target(struct cpufreq_policy *policy, - unsigned int target_freq, - unsigned int relation) -{ - unsigned int idx; - unsigned int freq; - int ret = 0; - - mutex_lock(&tegra_cpu_lock); - - if (is_suspended) { - ret = -EBUSY; - goto out; - } - - cpufreq_frequency_table_target(policy, freq_table, target_freq, - relation, &idx); - - freq = freq_table[idx].frequency; - - target_cpu_speed[policy->cpu] = freq; - - ret = tegra_update_cpu_speed(policy, tegra_cpu_highest_speed()); - -out: - mutex_unlock(&tegra_cpu_lock); - return ret; -} - -static int tegra_pm_notify(struct notifier_block *nb, unsigned long event, - void *dummy) -{ - mutex_lock(&tegra_cpu_lock); - if (event == PM_SUSPEND_PREPARE) { - struct cpufreq_policy *policy = cpufreq_cpu_get(0); - is_suspended = true; - pr_info("Tegra cpufreq suspend: setting frequency to %d kHz\n", - freq_table[0].frequency); - tegra_update_cpu_speed(policy, freq_table[0].frequency); - cpufreq_cpu_put(policy); - } else if (event == PM_POST_SUSPEND) { - is_suspended = false; - } - mutex_unlock(&tegra_cpu_lock); - - return NOTIFY_OK; -} - -static struct notifier_block tegra_cpu_pm_notifier = { - .notifier_call = tegra_pm_notify, -}; - -static int tegra_cpu_init(struct cpufreq_policy *policy) -{ - if (policy->cpu >= NUM_CPUS) - return -EINVAL; - - clk_prepare_enable(emc_clk); - clk_prepare_enable(cpu_clk); - - cpufreq_frequency_table_cpuinfo(policy, freq_table); - cpufreq_frequency_table_get_attr(freq_table, policy->cpu); - policy->cur = tegra_getspeed(policy->cpu); - target_cpu_speed[policy->cpu] = policy->cur; - - /* FIXME: what's the actual transition time? */ - policy->cpuinfo.transition_latency = 300 * 1000; - - cpumask_copy(policy->cpus, cpu_possible_mask); - - if (policy->cpu == 0) - register_pm_notifier(&tegra_cpu_pm_notifier); - - return 0; -} - -static int tegra_cpu_exit(struct cpufreq_policy *policy) -{ - cpufreq_frequency_table_cpuinfo(policy, freq_table); - clk_disable_unprepare(emc_clk); - return 0; -} - -static struct freq_attr *tegra_cpufreq_attr[] = { - &cpufreq_freq_attr_scaling_available_freqs, - NULL, -}; - -static struct cpufreq_driver tegra_cpufreq_driver = { - .verify = tegra_verify_speed, - .target = tegra_target, - .get = tegra_getspeed, - .init = tegra_cpu_init, - .exit = tegra_cpu_exit, - .name = "tegra", - .attr = tegra_cpufreq_attr, -}; - -static int __init tegra_cpufreq_init(void) -{ - cpu_clk = clk_get_sys(NULL, "cpu"); - if (IS_ERR(cpu_clk)) - return PTR_ERR(cpu_clk); - - pll_x_clk = clk_get_sys(NULL, "pll_x"); - if (IS_ERR(pll_x_clk)) - return PTR_ERR(pll_x_clk); - - pll_p_clk = clk_get_sys(NULL, "pll_p_cclk"); - if (IS_ERR(pll_p_clk)) - return PTR_ERR(pll_p_clk); - - emc_clk = clk_get_sys("cpu", "emc"); - if (IS_ERR(emc_clk)) { - clk_put(cpu_clk); - return PTR_ERR(emc_clk); - } - - return cpufreq_register_driver(&tegra_cpufreq_driver); -} - -static void __exit tegra_cpufreq_exit(void) -{ - cpufreq_unregister_driver(&tegra_cpufreq_driver); - clk_put(emc_clk); - clk_put(cpu_clk); -} - - -MODULE_AUTHOR("Colin Cross <ccross@android.com>"); -MODULE_DESCRIPTION("cpufreq driver for Nvidia Tegra2"); -MODULE_LICENSE("GPL"); -module_init(tegra_cpufreq_init); -module_exit(tegra_cpufreq_exit); diff --git a/drivers/cpufreq/tegra124-cpufreq.c b/drivers/cpufreq/tegra124-cpufreq.c new file mode 100644 index 000000000000..f8a76bbecef9 --- /dev/null +++ b/drivers/cpufreq/tegra124-cpufreq.c @@ -0,0 +1,240 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Tegra 124 cpufreq driver + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/clk.h> +#include <linux/cpufreq.h> +#include <linux/err.h> +#include <linux/init.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/platform_device.h> +#include <linux/pm_opp.h> +#include <linux/types.h> + +#include "cpufreq-dt.h" + +static struct platform_device *tegra124_cpufreq_pdev; + +struct tegra124_cpufreq_priv { + struct clk *cpu_clk; + struct clk *pllp_clk; + struct clk *pllx_clk; + struct clk *dfll_clk; + struct platform_device *cpufreq_dt_pdev; +}; + +static int tegra124_cpu_switch_to_dfll(struct tegra124_cpufreq_priv *priv) +{ + struct clk *orig_parent; + int ret; + + ret = clk_set_rate(priv->dfll_clk, clk_get_rate(priv->cpu_clk)); + if (ret) + return ret; + + orig_parent = clk_get_parent(priv->cpu_clk); + clk_set_parent(priv->cpu_clk, priv->pllp_clk); + + ret = clk_prepare_enable(priv->dfll_clk); + if (ret) + goto out; + + clk_set_parent(priv->cpu_clk, priv->dfll_clk); + + return 0; + +out: + clk_set_parent(priv->cpu_clk, orig_parent); + + return ret; +} + +static int tegra124_cpufreq_probe(struct platform_device *pdev) +{ + struct device_node *np __free(device_node) = of_cpu_device_node_get(0); + struct tegra124_cpufreq_priv *priv; + struct device *cpu_dev; + int ret; + + if (!np) + return -ENODEV; + + priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + cpu_dev = get_cpu_device(0); + if (!cpu_dev) + return -ENODEV; + + priv->cpu_clk = of_clk_get_by_name(np, "cpu_g"); + if (IS_ERR(priv->cpu_clk)) + return PTR_ERR(priv->cpu_clk); + + priv->dfll_clk = of_clk_get_by_name(np, "dfll"); + if (IS_ERR(priv->dfll_clk)) { + ret = PTR_ERR(priv->dfll_clk); + goto out_put_cpu_clk; + } + + priv->pllx_clk = of_clk_get_by_name(np, "pll_x"); + if (IS_ERR(priv->pllx_clk)) { + ret = PTR_ERR(priv->pllx_clk); + goto out_put_dfll_clk; + } + + priv->pllp_clk = of_clk_get_by_name(np, "pll_p"); + if (IS_ERR(priv->pllp_clk)) { + ret = PTR_ERR(priv->pllp_clk); + goto out_put_pllx_clk; + } + + ret = tegra124_cpu_switch_to_dfll(priv); + if (ret) + goto out_put_pllp_clk; + + priv->cpufreq_dt_pdev = cpufreq_dt_pdev_register(&pdev->dev); + if (IS_ERR(priv->cpufreq_dt_pdev)) { + ret = PTR_ERR(priv->cpufreq_dt_pdev); + goto out_put_pllp_clk; + } + + platform_set_drvdata(pdev, priv); + + return 0; + +out_put_pllp_clk: + clk_put(priv->pllp_clk); +out_put_pllx_clk: + clk_put(priv->pllx_clk); +out_put_dfll_clk: + clk_put(priv->dfll_clk); +out_put_cpu_clk: + clk_put(priv->cpu_clk); + + return ret; +} + +static int __maybe_unused tegra124_cpufreq_suspend(struct device *dev) +{ + struct tegra124_cpufreq_priv *priv = dev_get_drvdata(dev); + int err; + + /* + * PLLP rate 408Mhz is below the CPU Fmax at Vmin and is safe to + * use during suspend and resume. So, switch the CPU clock source + * to PLLP and disable DFLL. + */ + err = clk_set_parent(priv->cpu_clk, priv->pllp_clk); + if (err < 0) { + dev_err(dev, "failed to reparent to PLLP: %d\n", err); + return err; + } + + clk_disable_unprepare(priv->dfll_clk); + + return 0; +} + +static int __maybe_unused tegra124_cpufreq_resume(struct device *dev) +{ + struct tegra124_cpufreq_priv *priv = dev_get_drvdata(dev); + int err; + + /* + * Warmboot code powers up the CPU with PLLP clock source. + * Enable DFLL clock and switch CPU clock source back to DFLL. + */ + err = clk_prepare_enable(priv->dfll_clk); + if (err < 0) { + dev_err(dev, "failed to enable DFLL clock for CPU: %d\n", err); + goto disable_cpufreq; + } + + err = clk_set_parent(priv->cpu_clk, priv->dfll_clk); + if (err < 0) { + dev_err(dev, "failed to reparent to DFLL clock: %d\n", err); + goto disable_dfll; + } + + return 0; + +disable_dfll: + clk_disable_unprepare(priv->dfll_clk); +disable_cpufreq: + disable_cpufreq(); + + return err; +} + +static void tegra124_cpufreq_remove(struct platform_device *pdev) +{ + struct tegra124_cpufreq_priv *priv = dev_get_drvdata(&pdev->dev); + + if (!IS_ERR(priv->cpufreq_dt_pdev)) { + platform_device_unregister(priv->cpufreq_dt_pdev); + priv->cpufreq_dt_pdev = ERR_PTR(-ENODEV); + } + + clk_put(priv->pllp_clk); + clk_put(priv->pllx_clk); + clk_put(priv->dfll_clk); + clk_put(priv->cpu_clk); +} + +static const struct dev_pm_ops tegra124_cpufreq_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(tegra124_cpufreq_suspend, + tegra124_cpufreq_resume) +}; + +static struct platform_driver tegra124_cpufreq_platdrv = { + .driver.name = "cpufreq-tegra124", + .driver.pm = &tegra124_cpufreq_pm_ops, + .probe = tegra124_cpufreq_probe, + .remove = tegra124_cpufreq_remove, +}; + +static int __init tegra_cpufreq_init(void) +{ + int ret; + + if (!(of_machine_is_compatible("nvidia,tegra114") || + of_machine_is_compatible("nvidia,tegra124") || + of_machine_is_compatible("nvidia,tegra210"))) + return -ENODEV; + + /* + * Platform driver+device required for handling EPROBE_DEFER with + * the regulator and the DFLL clock + */ + ret = platform_driver_register(&tegra124_cpufreq_platdrv); + if (ret) + return ret; + + tegra124_cpufreq_pdev = platform_device_register_simple("cpufreq-tegra124", -1, NULL, 0); + if (IS_ERR(tegra124_cpufreq_pdev)) { + platform_driver_unregister(&tegra124_cpufreq_platdrv); + return PTR_ERR(tegra124_cpufreq_pdev); + } + + return 0; +} +module_init(tegra_cpufreq_init); + +static void __exit tegra_cpufreq_module_exit(void) +{ + if (!IS_ERR_OR_NULL(tegra124_cpufreq_pdev)) + platform_device_unregister(tegra124_cpufreq_pdev); + + platform_driver_unregister(&tegra124_cpufreq_platdrv); +} +module_exit(tegra_cpufreq_module_exit); + +MODULE_AUTHOR("Tuomas Tynkkynen <ttynkkynen@nvidia.com>"); +MODULE_DESCRIPTION("cpufreq driver for NVIDIA Tegra124"); +MODULE_LICENSE("GPL"); diff --git a/drivers/cpufreq/tegra186-cpufreq.c b/drivers/cpufreq/tegra186-cpufreq.c new file mode 100644 index 000000000000..34ed943c5f34 --- /dev/null +++ b/drivers/cpufreq/tegra186-cpufreq.c @@ -0,0 +1,444 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved + */ + +#include <linux/cpufreq.h> +#include <linux/dma-mapping.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/platform_device.h> +#include <linux/units.h> + +#include <soc/tegra/bpmp.h> +#include <soc/tegra/bpmp-abi.h> + +#define TEGRA186_NUM_CLUSTERS 2 +#define EDVD_OFFSET_A57(core) ((SZ_64K * 6) + (0x20 + (core) * 0x4)) +#define EDVD_OFFSET_DENVER(core) ((SZ_64K * 7) + (0x20 + (core) * 0x4)) +#define EDVD_CORE_VOLT_FREQ_F_SHIFT 0 +#define EDVD_CORE_VOLT_FREQ_F_MASK 0xffff +#define EDVD_CORE_VOLT_FREQ_V_SHIFT 16 + +struct tegra186_cpufreq_cpu { + unsigned int bpmp_cluster_id; + unsigned int edvd_offset; +}; + +static const struct tegra186_cpufreq_cpu tegra186_cpus[] = { + /* CPU0 - A57 Cluster */ + { + .bpmp_cluster_id = 1, + .edvd_offset = EDVD_OFFSET_A57(0) + }, + /* CPU1 - Denver Cluster */ + { + .bpmp_cluster_id = 0, + .edvd_offset = EDVD_OFFSET_DENVER(0) + }, + /* CPU2 - Denver Cluster */ + { + .bpmp_cluster_id = 0, + .edvd_offset = EDVD_OFFSET_DENVER(1) + }, + /* CPU3 - A57 Cluster */ + { + .bpmp_cluster_id = 1, + .edvd_offset = EDVD_OFFSET_A57(1) + }, + /* CPU4 - A57 Cluster */ + { + .bpmp_cluster_id = 1, + .edvd_offset = EDVD_OFFSET_A57(2) + }, + /* CPU5 - A57 Cluster */ + { + .bpmp_cluster_id = 1, + .edvd_offset = EDVD_OFFSET_A57(3) + }, +}; + +struct tegra186_cpufreq_cluster { + struct cpufreq_frequency_table *bpmp_lut; + u32 ref_clk_khz; + u32 div; +}; + +struct tegra186_cpufreq_data { + void __iomem *regs; + const struct tegra186_cpufreq_cpu *cpus; + bool icc_dram_bw_scaling; + struct tegra186_cpufreq_cluster clusters[]; +}; + +static int tegra_cpufreq_set_bw(struct cpufreq_policy *policy, unsigned long freq_khz) +{ + struct tegra186_cpufreq_data *data = cpufreq_get_driver_data(); + struct device *dev; + int ret; + + dev = get_cpu_device(policy->cpu); + if (!dev) + return -ENODEV; + + struct dev_pm_opp *opp __free(put_opp) = + dev_pm_opp_find_freq_exact(dev, freq_khz * HZ_PER_KHZ, true); + if (IS_ERR(opp)) + return PTR_ERR(opp); + + ret = dev_pm_opp_set_opp(dev, opp); + if (ret) + data->icc_dram_bw_scaling = false; + + return ret; +} + +static int tegra_cpufreq_init_cpufreq_table(struct cpufreq_policy *policy, + struct cpufreq_frequency_table *bpmp_lut, + struct cpufreq_frequency_table **opp_table) +{ + struct tegra186_cpufreq_data *data = cpufreq_get_driver_data(); + struct cpufreq_frequency_table *freq_table = NULL; + struct cpufreq_frequency_table *pos; + struct device *cpu_dev; + unsigned long rate; + int ret, max_opps; + int j = 0; + + cpu_dev = get_cpu_device(policy->cpu); + if (!cpu_dev) { + pr_err("%s: failed to get cpu%d device\n", __func__, policy->cpu); + return -ENODEV; + } + + /* Initialize OPP table mentioned in operating-points-v2 property in DT */ + ret = dev_pm_opp_of_add_table_indexed(cpu_dev, 0); + if (ret) { + dev_err(cpu_dev, "Invalid or empty opp table in device tree\n"); + data->icc_dram_bw_scaling = false; + return ret; + } + + max_opps = dev_pm_opp_get_opp_count(cpu_dev); + if (max_opps <= 0) { + dev_err(cpu_dev, "Failed to add OPPs\n"); + return max_opps; + } + + /* Disable all opps and cross-validate against LUT later */ + for (rate = 0; ; rate++) { + struct dev_pm_opp *opp __free(put_opp) = + dev_pm_opp_find_freq_ceil(cpu_dev, &rate); + if (IS_ERR(opp)) + break; + + dev_pm_opp_disable(cpu_dev, rate); + } + + freq_table = kcalloc((max_opps + 1), sizeof(*freq_table), GFP_KERNEL); + if (!freq_table) + return -ENOMEM; + + /* + * Cross check the frequencies from BPMP-FW LUT against the OPP's present in DT. + * Enable only those DT OPP's which are present in LUT also. + */ + cpufreq_for_each_valid_entry(pos, bpmp_lut) { + struct dev_pm_opp *opp __free(put_opp) = + dev_pm_opp_find_freq_exact(cpu_dev, pos->frequency * HZ_PER_KHZ, false); + if (IS_ERR(opp)) + continue; + + ret = dev_pm_opp_enable(cpu_dev, pos->frequency * HZ_PER_KHZ); + if (ret < 0) + return ret; + + freq_table[j].driver_data = pos->driver_data; + freq_table[j].frequency = pos->frequency; + j++; + } + + freq_table[j].driver_data = pos->driver_data; + freq_table[j].frequency = CPUFREQ_TABLE_END; + + *opp_table = &freq_table[0]; + + dev_pm_opp_set_sharing_cpus(cpu_dev, policy->cpus); + + /* Prime interconnect data */ + tegra_cpufreq_set_bw(policy, freq_table[j - 1].frequency); + + return ret; +} + +static int tegra186_cpufreq_init(struct cpufreq_policy *policy) +{ + struct tegra186_cpufreq_data *data = cpufreq_get_driver_data(); + unsigned int cluster = data->cpus[policy->cpu].bpmp_cluster_id; + struct cpufreq_frequency_table *freq_table; + struct cpufreq_frequency_table *bpmp_lut; + u32 cpu; + int ret; + + policy->cpuinfo.transition_latency = 300 * 1000; + policy->driver_data = NULL; + + /* set same policy for all cpus in a cluster */ + for (cpu = 0; cpu < ARRAY_SIZE(tegra186_cpus); cpu++) { + if (data->cpus[cpu].bpmp_cluster_id == cluster) + cpumask_set_cpu(cpu, policy->cpus); + } + + bpmp_lut = data->clusters[cluster].bpmp_lut; + + if (data->icc_dram_bw_scaling) { + ret = tegra_cpufreq_init_cpufreq_table(policy, bpmp_lut, &freq_table); + if (!ret) { + policy->freq_table = freq_table; + return 0; + } + } + + data->icc_dram_bw_scaling = false; + policy->freq_table = bpmp_lut; + pr_info("OPP tables missing from DT, EMC frequency scaling disabled\n"); + + return 0; +} + +static int tegra186_cpufreq_set_target(struct cpufreq_policy *policy, + unsigned int index) +{ + struct tegra186_cpufreq_data *data = cpufreq_get_driver_data(); + struct cpufreq_frequency_table *tbl = policy->freq_table + index; + unsigned int edvd_offset; + u32 edvd_val = tbl->driver_data; + u32 cpu; + + for_each_cpu(cpu, policy->cpus) { + edvd_offset = data->cpus[cpu].edvd_offset; + writel(edvd_val, data->regs + edvd_offset); + } + + if (data->icc_dram_bw_scaling) + tegra_cpufreq_set_bw(policy, tbl->frequency); + + + return 0; +} + +static unsigned int tegra186_cpufreq_get(unsigned int cpu) +{ + struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu); + struct tegra186_cpufreq_data *data = cpufreq_get_driver_data(); + struct tegra186_cpufreq_cluster *cluster; + unsigned int edvd_offset, cluster_id; + u32 ndiv; + + if (!policy) + return 0; + + edvd_offset = data->cpus[policy->cpu].edvd_offset; + ndiv = readl(data->regs + edvd_offset) & EDVD_CORE_VOLT_FREQ_F_MASK; + cluster_id = data->cpus[policy->cpu].bpmp_cluster_id; + cluster = &data->clusters[cluster_id]; + + return (cluster->ref_clk_khz * ndiv) / cluster->div; +} + +static struct cpufreq_driver tegra186_cpufreq_driver = { + .name = "tegra186", + .flags = CPUFREQ_HAVE_GOVERNOR_PER_POLICY | + CPUFREQ_NEED_INITIAL_FREQ_CHECK, + .get = tegra186_cpufreq_get, + .verify = cpufreq_generic_frequency_table_verify, + .target_index = tegra186_cpufreq_set_target, + .init = tegra186_cpufreq_init, +}; + +static struct cpufreq_frequency_table *tegra_cpufreq_bpmp_read_lut( + struct platform_device *pdev, struct tegra_bpmp *bpmp, + struct tegra186_cpufreq_cluster *cluster, unsigned int cluster_id, + int *num_rates) +{ + struct cpufreq_frequency_table *table; + struct mrq_cpu_vhint_request req; + struct tegra_bpmp_message msg; + struct cpu_vhint_data *data; + int err, i, j; + dma_addr_t phys; + void *virt; + + virt = dma_alloc_coherent(bpmp->dev, sizeof(*data), &phys, + GFP_KERNEL); + if (!virt) + return ERR_PTR(-ENOMEM); + + data = (struct cpu_vhint_data *)virt; + + memset(&req, 0, sizeof(req)); + req.addr = phys; + req.cluster_id = cluster_id; + + memset(&msg, 0, sizeof(msg)); + msg.mrq = MRQ_CPU_VHINT; + msg.tx.data = &req; + msg.tx.size = sizeof(req); + + err = tegra_bpmp_transfer(bpmp, &msg); + if (err) { + table = ERR_PTR(err); + goto free; + } + if (msg.rx.ret) { + table = ERR_PTR(-EINVAL); + goto free; + } + + *num_rates = 0; + for (i = data->vfloor; i <= data->vceil; i++) { + u16 ndiv = data->ndiv[i]; + + if (ndiv < data->ndiv_min || ndiv > data->ndiv_max) + continue; + + /* Only store lowest voltage index for each rate */ + if (i > 0 && ndiv == data->ndiv[i - 1]) + continue; + + (*num_rates)++; + } + + table = devm_kcalloc(&pdev->dev, *num_rates + 1, sizeof(*table), + GFP_KERNEL); + if (!table) { + table = ERR_PTR(-ENOMEM); + goto free; + } + + cluster->ref_clk_khz = data->ref_clk_hz / 1000; + cluster->div = data->pdiv * data->mdiv; + + for (i = data->vfloor, j = 0; i <= data->vceil; i++) { + struct cpufreq_frequency_table *point; + u16 ndiv = data->ndiv[i]; + u32 edvd_val = 0; + + if (ndiv < data->ndiv_min || ndiv > data->ndiv_max) + continue; + + /* Only store lowest voltage index for each rate */ + if (i > 0 && ndiv == data->ndiv[i - 1]) + continue; + + edvd_val |= i << EDVD_CORE_VOLT_FREQ_V_SHIFT; + edvd_val |= ndiv << EDVD_CORE_VOLT_FREQ_F_SHIFT; + + point = &table[j++]; + point->driver_data = edvd_val; + point->frequency = (cluster->ref_clk_khz * ndiv) / cluster->div; + } + + table[j].frequency = CPUFREQ_TABLE_END; + +free: + dma_free_coherent(bpmp->dev, sizeof(*data), virt, phys); + + return table; +} + +static int tegra186_cpufreq_probe(struct platform_device *pdev) +{ + struct tegra186_cpufreq_data *data; + struct tegra_bpmp *bpmp; + struct device *cpu_dev; + unsigned int i = 0, err, edvd_offset; + int num_rates = 0; + u32 edvd_val, cpu; + + data = devm_kzalloc(&pdev->dev, + struct_size(data, clusters, TEGRA186_NUM_CLUSTERS), + GFP_KERNEL); + if (!data) + return -ENOMEM; + + data->cpus = tegra186_cpus; + + bpmp = tegra_bpmp_get(&pdev->dev); + if (IS_ERR(bpmp)) + return PTR_ERR(bpmp); + + data->regs = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(data->regs)) { + err = PTR_ERR(data->regs); + goto put_bpmp; + } + + for (i = 0; i < TEGRA186_NUM_CLUSTERS; i++) { + struct tegra186_cpufreq_cluster *cluster = &data->clusters[i]; + + cluster->bpmp_lut = tegra_cpufreq_bpmp_read_lut(pdev, bpmp, cluster, i, &num_rates); + if (IS_ERR(cluster->bpmp_lut)) { + err = PTR_ERR(cluster->bpmp_lut); + goto put_bpmp; + } else if (!num_rates) { + err = -EINVAL; + goto put_bpmp; + } + + for (cpu = 0; cpu < ARRAY_SIZE(tegra186_cpus); cpu++) { + if (data->cpus[cpu].bpmp_cluster_id == i) { + edvd_val = cluster->bpmp_lut[num_rates - 1].driver_data; + edvd_offset = data->cpus[cpu].edvd_offset; + writel(edvd_val, data->regs + edvd_offset); + } + } + } + + tegra186_cpufreq_driver.driver_data = data; + + /* Check for optional OPPv2 and interconnect paths on CPU0 to enable ICC scaling */ + cpu_dev = get_cpu_device(0); + if (!cpu_dev) { + err = -EPROBE_DEFER; + goto put_bpmp; + } + + if (dev_pm_opp_of_get_opp_desc_node(cpu_dev)) { + err = dev_pm_opp_of_find_icc_paths(cpu_dev, NULL); + if (!err) + data->icc_dram_bw_scaling = true; + } + + err = cpufreq_register_driver(&tegra186_cpufreq_driver); + +put_bpmp: + tegra_bpmp_put(bpmp); + + return err; +} + +static void tegra186_cpufreq_remove(struct platform_device *pdev) +{ + cpufreq_unregister_driver(&tegra186_cpufreq_driver); +} + +static const struct of_device_id tegra186_cpufreq_of_match[] = { + { .compatible = "nvidia,tegra186-ccplex-cluster", }, + { } +}; +MODULE_DEVICE_TABLE(of, tegra186_cpufreq_of_match); + +static struct platform_driver tegra186_cpufreq_platform_driver = { + .driver = { + .name = "tegra186-cpufreq", + .of_match_table = tegra186_cpufreq_of_match, + }, + .probe = tegra186_cpufreq_probe, + .remove = tegra186_cpufreq_remove, +}; +module_platform_driver(tegra186_cpufreq_platform_driver); + +MODULE_AUTHOR("Mikko Perttunen <mperttunen@nvidia.com>"); +MODULE_DESCRIPTION("NVIDIA Tegra186 cpufreq driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/cpufreq/tegra194-cpufreq.c b/drivers/cpufreq/tegra194-cpufreq.c new file mode 100644 index 000000000000..695599e1001f --- /dev/null +++ b/drivers/cpufreq/tegra194-cpufreq.c @@ -0,0 +1,828 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2020 - 2022, NVIDIA CORPORATION. All rights reserved + */ + +#include <linux/cpu.h> +#include <linux/cpufreq.h> +#include <linux/dma-mapping.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_platform.h> +#include <linux/platform_device.h> +#include <linux/slab.h> +#include <linux/units.h> + +#include <asm/smp_plat.h> + +#include <soc/tegra/bpmp.h> +#include <soc/tegra/bpmp-abi.h> + +#define KHZ 1000 +#define REF_CLK_MHZ 408 /* 408 MHz */ +#define CPUFREQ_TBL_STEP_HZ (50 * KHZ * KHZ) +#define MAX_CNT ~0U + +#define MAX_DELTA_KHZ 115200 + +#define NDIV_MASK 0x1FF + +#define CORE_OFFSET(cpu) (cpu * 8) +#define CMU_CLKS_BASE 0x2000 +#define SCRATCH_FREQ_CORE_REG(data, cpu) (data->regs + CMU_CLKS_BASE + CORE_OFFSET(cpu)) + +#define MMCRAB_CLUSTER_BASE(cl) (0x30000 + (cl * 0x10000)) +#define CLUSTER_ACTMON_BASE(data, cl) \ + (data->regs + (MMCRAB_CLUSTER_BASE(cl) + data->soc->actmon_cntr_base)) +#define CORE_ACTMON_CNTR_REG(data, cl, cpu) (CLUSTER_ACTMON_BASE(data, cl) + CORE_OFFSET(cpu)) + +/* cpufreq transisition latency */ +#define TEGRA_CPUFREQ_TRANSITION_LATENCY (300 * 1000) /* unit in nanoseconds */ + +struct tegra_cpu_data { + u32 cpuid; + u32 clusterid; + void __iomem *freq_core_reg; +}; + +struct tegra_cpu_ctr { + u32 cpu; + u32 coreclk_cnt, last_coreclk_cnt; + u32 refclk_cnt, last_refclk_cnt; +}; + +struct read_counters_work { + struct work_struct work; + struct tegra_cpu_ctr c; +}; + +struct tegra_cpufreq_ops { + void (*read_counters)(struct tegra_cpu_ctr *c); + void (*set_cpu_ndiv)(struct cpufreq_policy *policy, u64 ndiv); + void (*get_cpu_cluster_id)(u32 cpu, u32 *cpuid, u32 *clusterid); + int (*get_cpu_ndiv)(u32 cpu, u32 cpuid, u32 clusterid, u64 *ndiv); +}; + +struct tegra_cpufreq_soc { + struct tegra_cpufreq_ops *ops; + int maxcpus_per_cluster; + unsigned int num_clusters; + phys_addr_t actmon_cntr_base; + u32 refclk_delta_min; +}; + +struct tegra194_cpufreq_data { + void __iomem *regs; + struct cpufreq_frequency_table **bpmp_luts; + const struct tegra_cpufreq_soc *soc; + bool icc_dram_bw_scaling; + struct tegra_cpu_data *cpu_data; +}; + +static struct workqueue_struct *read_counters_wq; + +static int tegra_cpufreq_set_bw(struct cpufreq_policy *policy, unsigned long freq_khz) +{ + struct tegra194_cpufreq_data *data = cpufreq_get_driver_data(); + struct dev_pm_opp *opp; + struct device *dev; + int ret; + + dev = get_cpu_device(policy->cpu); + if (!dev) + return -ENODEV; + + opp = dev_pm_opp_find_freq_exact(dev, freq_khz * KHZ, true); + if (IS_ERR(opp)) + return PTR_ERR(opp); + + ret = dev_pm_opp_set_opp(dev, opp); + if (ret) + data->icc_dram_bw_scaling = false; + + dev_pm_opp_put(opp); + return ret; +} + +static void tegra_get_cpu_mpidr(void *mpidr) +{ + *((u64 *)mpidr) = read_cpuid_mpidr() & MPIDR_HWID_BITMASK; +} + +static void tegra234_get_cpu_cluster_id(u32 cpu, u32 *cpuid, u32 *clusterid) +{ + u64 mpidr; + + smp_call_function_single(cpu, tegra_get_cpu_mpidr, &mpidr, true); + + if (cpuid) + *cpuid = MPIDR_AFFINITY_LEVEL(mpidr, 1); + if (clusterid) + *clusterid = MPIDR_AFFINITY_LEVEL(mpidr, 2); +} + +static int tegra234_get_cpu_ndiv(u32 cpu, u32 cpuid, u32 clusterid, u64 *ndiv) +{ + struct tegra194_cpufreq_data *data = cpufreq_get_driver_data(); + + *ndiv = readl(data->cpu_data[cpu].freq_core_reg) & NDIV_MASK; + + return 0; +} + +static void tegra234_set_cpu_ndiv(struct cpufreq_policy *policy, u64 ndiv) +{ + struct tegra194_cpufreq_data *data = cpufreq_get_driver_data(); + u32 cpu; + + for_each_cpu(cpu, policy->cpus) + writel(ndiv, data->cpu_data[cpu].freq_core_reg); +} + +/* + * This register provides access to two counter values with a single + * 64-bit read. The counter values are used to determine the average + * actual frequency a core has run at over a period of time. + * [63:32] PLLP counter: Counts at fixed frequency (408 MHz) + * [31:0] Core clock counter: Counts on every core clock cycle + */ +static void tegra234_read_counters(struct tegra_cpu_ctr *c) +{ + struct tegra194_cpufreq_data *data = cpufreq_get_driver_data(); + void __iomem *actmon_reg; + u32 delta_refcnt; + int cnt = 0; + u64 val; + + actmon_reg = CORE_ACTMON_CNTR_REG(data, data->cpu_data[c->cpu].clusterid, + data->cpu_data[c->cpu].cpuid); + + val = readq(actmon_reg); + c->last_refclk_cnt = upper_32_bits(val); + c->last_coreclk_cnt = lower_32_bits(val); + + /* + * The sampling window is based on the minimum number of reference + * clock cycles which is known to give a stable value of CPU frequency. + */ + do { + val = readq(actmon_reg); + c->refclk_cnt = upper_32_bits(val); + c->coreclk_cnt = lower_32_bits(val); + if (c->refclk_cnt < c->last_refclk_cnt) + delta_refcnt = c->refclk_cnt + (MAX_CNT - c->last_refclk_cnt); + else + delta_refcnt = c->refclk_cnt - c->last_refclk_cnt; + if (++cnt >= 0xFFFF) { + pr_warn("cpufreq: problem with refclk on cpu:%d, delta_refcnt:%u, cnt:%d\n", + c->cpu, delta_refcnt, cnt); + break; + } + } while (delta_refcnt < data->soc->refclk_delta_min); +} + +static struct tegra_cpufreq_ops tegra234_cpufreq_ops = { + .read_counters = tegra234_read_counters, + .get_cpu_cluster_id = tegra234_get_cpu_cluster_id, + .get_cpu_ndiv = tegra234_get_cpu_ndiv, + .set_cpu_ndiv = tegra234_set_cpu_ndiv, +}; + +static const struct tegra_cpufreq_soc tegra234_cpufreq_soc = { + .ops = &tegra234_cpufreq_ops, + .actmon_cntr_base = 0x9000, + .maxcpus_per_cluster = 4, + .num_clusters = 3, + .refclk_delta_min = 16000, +}; + +static const struct tegra_cpufreq_soc tegra239_cpufreq_soc = { + .ops = &tegra234_cpufreq_ops, + .actmon_cntr_base = 0x4000, + .maxcpus_per_cluster = 8, + .num_clusters = 1, + .refclk_delta_min = 16000, +}; + +static void tegra194_get_cpu_cluster_id(u32 cpu, u32 *cpuid, u32 *clusterid) +{ + u64 mpidr; + + smp_call_function_single(cpu, tegra_get_cpu_mpidr, &mpidr, true); + + if (cpuid) + *cpuid = MPIDR_AFFINITY_LEVEL(mpidr, 0); + if (clusterid) + *clusterid = MPIDR_AFFINITY_LEVEL(mpidr, 1); +} + +/* + * Read per-core Read-only system register NVFREQ_FEEDBACK_EL1. + * The register provides frequency feedback information to + * determine the average actual frequency a core has run at over + * a period of time. + * [31:0] PLLP counter: Counts at fixed frequency (408 MHz) + * [63:32] Core clock counter: counts on every core clock cycle + * where the core is architecturally clocking + */ +static u64 read_freq_feedback(void) +{ + u64 val = 0; + + asm volatile("mrs %0, s3_0_c15_c0_5" : "=r" (val) : ); + + return val; +} + +static inline u32 map_ndiv_to_freq(struct mrq_cpu_ndiv_limits_response + *nltbl, u16 ndiv) +{ + return nltbl->ref_clk_hz / KHZ * ndiv / (nltbl->pdiv * nltbl->mdiv); +} + +static void tegra194_read_counters(struct tegra_cpu_ctr *c) +{ + struct tegra194_cpufreq_data *data = cpufreq_get_driver_data(); + u32 delta_refcnt; + int cnt = 0; + u64 val; + + val = read_freq_feedback(); + c->last_refclk_cnt = lower_32_bits(val); + c->last_coreclk_cnt = upper_32_bits(val); + + /* + * The sampling window is based on the minimum number of reference + * clock cycles which is known to give a stable value of CPU frequency. + */ + do { + val = read_freq_feedback(); + c->refclk_cnt = lower_32_bits(val); + c->coreclk_cnt = upper_32_bits(val); + if (c->refclk_cnt < c->last_refclk_cnt) + delta_refcnt = c->refclk_cnt + (MAX_CNT - c->last_refclk_cnt); + else + delta_refcnt = c->refclk_cnt - c->last_refclk_cnt; + if (++cnt >= 0xFFFF) { + pr_warn("cpufreq: problem with refclk on cpu:%d, delta_refcnt:%u, cnt:%d\n", + c->cpu, delta_refcnt, cnt); + break; + } + } while (delta_refcnt < data->soc->refclk_delta_min); +} + +static void tegra_read_counters(struct work_struct *work) +{ + struct tegra194_cpufreq_data *data = cpufreq_get_driver_data(); + struct read_counters_work *read_counters_work; + struct tegra_cpu_ctr *c; + + /* + * ref_clk_counter(32 bit counter) runs on constant clk, + * pll_p(408MHz). + * It will take = 2 ^ 32 / 408 MHz to overflow ref clk counter + * = 10526880 usec = 10.527 sec to overflow + * + * Like wise core_clk_counter(32 bit counter) runs on core clock. + * It's synchronized to crab_clk (cpu_crab_clk) which runs at + * freq of cluster. Assuming max cluster clock ~2000MHz, + * It will take = 2 ^ 32 / 2000 MHz to overflow core clk counter + * = ~2.147 sec to overflow + */ + read_counters_work = container_of(work, struct read_counters_work, + work); + c = &read_counters_work->c; + + data->soc->ops->read_counters(c); +} + +/* + * Return instantaneous cpu speed + * Instantaneous freq is calculated as - + * -Takes sample on every query of getting the freq. + * - Read core and ref clock counters; + * - Delay for X us + * - Read above cycle counters again + * - Calculates freq by subtracting current and previous counters + * divided by the delay time or eqv. of ref_clk_counter in delta time + * - Return Kcycles/second, freq in KHz + * + * delta time period = x sec + * = delta ref_clk_counter / (408 * 10^6) sec + * freq in Hz = cycles/sec + * = (delta cycles / x sec + * = (delta cycles * 408 * 10^6) / delta ref_clk_counter + * in KHz = (delta cycles * 408 * 10^3) / delta ref_clk_counter + * + * @cpu - logical cpu whose freq to be updated + * Returns freq in KHz on success, 0 if cpu is offline + */ +static unsigned int tegra194_calculate_speed(u32 cpu) +{ + struct read_counters_work read_counters_work; + struct tegra_cpu_ctr c; + u32 delta_refcnt; + u32 delta_ccnt; + u32 rate_mhz; + + /* + * Reconstruct cpu frequency over an observation/sampling window. + * Using workqueue to keep interrupts enabled during the interval. + */ + read_counters_work.c.cpu = cpu; + INIT_WORK_ONSTACK(&read_counters_work.work, tegra_read_counters); + queue_work_on(cpu, read_counters_wq, &read_counters_work.work); + flush_work(&read_counters_work.work); + c = read_counters_work.c; + + if (c.coreclk_cnt < c.last_coreclk_cnt) + delta_ccnt = c.coreclk_cnt + (MAX_CNT - c.last_coreclk_cnt); + else + delta_ccnt = c.coreclk_cnt - c.last_coreclk_cnt; + if (!delta_ccnt) + return 0; + + /* ref clock is 32 bits */ + if (c.refclk_cnt < c.last_refclk_cnt) + delta_refcnt = c.refclk_cnt + (MAX_CNT - c.last_refclk_cnt); + else + delta_refcnt = c.refclk_cnt - c.last_refclk_cnt; + if (!delta_refcnt) { + pr_debug("cpufreq: %d is idle, delta_refcnt: 0\n", cpu); + return 0; + } + rate_mhz = ((unsigned long)(delta_ccnt * REF_CLK_MHZ)) / delta_refcnt; + + return (rate_mhz * KHZ); /* in KHz */ +} + +static void tegra194_get_cpu_ndiv_sysreg(void *ndiv) +{ + u64 ndiv_val; + + asm volatile("mrs %0, s3_0_c15_c0_4" : "=r" (ndiv_val) : ); + + *(u64 *)ndiv = ndiv_val; +} + +static int tegra194_get_cpu_ndiv(u32 cpu, u32 cpuid, u32 clusterid, u64 *ndiv) +{ + return smp_call_function_single(cpu, tegra194_get_cpu_ndiv_sysreg, &ndiv, true); +} + +static void tegra194_set_cpu_ndiv_sysreg(void *data) +{ + u64 ndiv_val = *(u64 *)data; + + asm volatile("msr s3_0_c15_c0_4, %0" : : "r" (ndiv_val)); +} + +static void tegra194_set_cpu_ndiv(struct cpufreq_policy *policy, u64 ndiv) +{ + on_each_cpu_mask(policy->cpus, tegra194_set_cpu_ndiv_sysreg, &ndiv, true); +} + +static unsigned int tegra194_get_speed(u32 cpu) +{ + struct tegra194_cpufreq_data *data = cpufreq_get_driver_data(); + u32 clusterid = data->cpu_data[cpu].clusterid; + struct cpufreq_frequency_table *pos; + unsigned int rate; + u64 ndiv; + int ret; + + /* reconstruct actual cpu freq using counters */ + rate = tegra194_calculate_speed(cpu); + + /* get last written ndiv value */ + ret = data->soc->ops->get_cpu_ndiv(cpu, data->cpu_data[cpu].cpuid, clusterid, &ndiv); + if (WARN_ON_ONCE(ret)) + return rate; + + /* + * If the reconstructed frequency has acceptable delta from + * the last written value, then return freq corresponding + * to the last written ndiv value from freq_table. This is + * done to return consistent value. + */ + cpufreq_for_each_valid_entry(pos, data->bpmp_luts[clusterid]) { + if (pos->driver_data != ndiv) + continue; + + if (abs(pos->frequency - rate) > MAX_DELTA_KHZ) { + pr_warn("cpufreq: cpu%d,cur:%u,set:%u,delta:%d,set ndiv:%llu\n", + cpu, rate, pos->frequency, abs(rate - pos->frequency), ndiv); + } else { + rate = pos->frequency; + } + break; + } + return rate; +} + +static int tegra_cpufreq_init_cpufreq_table(struct cpufreq_policy *policy, + struct cpufreq_frequency_table *bpmp_lut, + struct cpufreq_frequency_table **opp_table) +{ + struct tegra194_cpufreq_data *data = cpufreq_get_driver_data(); + struct cpufreq_frequency_table *freq_table = NULL; + struct cpufreq_frequency_table *pos; + struct device *cpu_dev; + struct dev_pm_opp *opp; + unsigned long rate; + int ret, max_opps; + int j = 0; + + cpu_dev = get_cpu_device(policy->cpu); + if (!cpu_dev) { + pr_err("%s: failed to get cpu%d device\n", __func__, policy->cpu); + return -ENODEV; + } + + /* Initialize OPP table mentioned in operating-points-v2 property in DT */ + ret = dev_pm_opp_of_add_table_indexed(cpu_dev, 0); + if (!ret) { + max_opps = dev_pm_opp_get_opp_count(cpu_dev); + if (max_opps <= 0) { + dev_err(cpu_dev, "Failed to add OPPs\n"); + return max_opps; + } + + /* Disable all opps and cross-validate against LUT later */ + for (rate = 0; ; rate++) { + opp = dev_pm_opp_find_freq_ceil(cpu_dev, &rate); + if (IS_ERR(opp)) + break; + + dev_pm_opp_put(opp); + dev_pm_opp_disable(cpu_dev, rate); + } + } else { + dev_err(cpu_dev, "Invalid or empty opp table in device tree\n"); + data->icc_dram_bw_scaling = false; + return ret; + } + + freq_table = kcalloc((max_opps + 1), sizeof(*freq_table), GFP_KERNEL); + if (!freq_table) + return -ENOMEM; + + /* + * Cross check the frequencies from BPMP-FW LUT against the OPP's present in DT. + * Enable only those DT OPP's which are present in LUT also. + */ + cpufreq_for_each_valid_entry(pos, bpmp_lut) { + opp = dev_pm_opp_find_freq_exact(cpu_dev, pos->frequency * KHZ, false); + if (IS_ERR(opp)) + continue; + + dev_pm_opp_put(opp); + + ret = dev_pm_opp_enable(cpu_dev, pos->frequency * KHZ); + if (ret < 0) + return ret; + + freq_table[j].driver_data = pos->driver_data; + freq_table[j].frequency = pos->frequency; + j++; + } + + freq_table[j].driver_data = pos->driver_data; + freq_table[j].frequency = CPUFREQ_TABLE_END; + + *opp_table = &freq_table[0]; + + dev_pm_opp_set_sharing_cpus(cpu_dev, policy->cpus); + + return ret; +} + +static int tegra194_cpufreq_init(struct cpufreq_policy *policy) +{ + struct tegra194_cpufreq_data *data = cpufreq_get_driver_data(); + int maxcpus_per_cluster = data->soc->maxcpus_per_cluster; + u32 clusterid = data->cpu_data[policy->cpu].clusterid; + struct cpufreq_frequency_table *freq_table; + struct cpufreq_frequency_table *bpmp_lut; + u32 start_cpu, cpu; + int ret; + + if (clusterid >= data->soc->num_clusters || !data->bpmp_luts[clusterid]) + return -EINVAL; + + start_cpu = rounddown(policy->cpu, maxcpus_per_cluster); + /* set same policy for all cpus in a cluster */ + for (cpu = start_cpu; cpu < (start_cpu + maxcpus_per_cluster); cpu++) { + if (cpu_possible(cpu)) + cpumask_set_cpu(cpu, policy->cpus); + } + policy->cpuinfo.transition_latency = TEGRA_CPUFREQ_TRANSITION_LATENCY; + + bpmp_lut = data->bpmp_luts[clusterid]; + + if (data->icc_dram_bw_scaling) { + ret = tegra_cpufreq_init_cpufreq_table(policy, bpmp_lut, &freq_table); + if (!ret) { + policy->freq_table = freq_table; + return 0; + } + } + + data->icc_dram_bw_scaling = false; + policy->freq_table = bpmp_lut; + pr_info("OPP tables missing from DT, EMC frequency scaling disabled\n"); + + return 0; +} + +static int tegra194_cpufreq_online(struct cpufreq_policy *policy) +{ + /* We did light-weight tear down earlier, nothing to do here */ + return 0; +} + +static int tegra194_cpufreq_offline(struct cpufreq_policy *policy) +{ + /* + * Preserve policy->driver_data and don't free resources on light-weight + * tear down. + */ + + return 0; +} + +static void tegra194_cpufreq_exit(struct cpufreq_policy *policy) +{ + struct device *cpu_dev = get_cpu_device(policy->cpu); + + dev_pm_opp_remove_all_dynamic(cpu_dev); + dev_pm_opp_of_cpumask_remove_table(policy->related_cpus); +} + +static int tegra194_cpufreq_set_target(struct cpufreq_policy *policy, + unsigned int index) +{ + struct cpufreq_frequency_table *tbl = policy->freq_table + index; + struct tegra194_cpufreq_data *data = cpufreq_get_driver_data(); + + /* + * Each core writes frequency in per core register. Then both cores + * in a cluster run at same frequency which is the maximum frequency + * request out of the values requested by both cores in that cluster. + */ + data->soc->ops->set_cpu_ndiv(policy, (u64)tbl->driver_data); + + if (data->icc_dram_bw_scaling) + tegra_cpufreq_set_bw(policy, tbl->frequency); + + return 0; +} + +static struct cpufreq_driver tegra194_cpufreq_driver = { + .name = "tegra194", + .flags = CPUFREQ_CONST_LOOPS | CPUFREQ_NEED_INITIAL_FREQ_CHECK | + CPUFREQ_IS_COOLING_DEV, + .verify = cpufreq_generic_frequency_table_verify, + .target_index = tegra194_cpufreq_set_target, + .get = tegra194_get_speed, + .init = tegra194_cpufreq_init, + .exit = tegra194_cpufreq_exit, + .online = tegra194_cpufreq_online, + .offline = tegra194_cpufreq_offline, +}; + +static struct tegra_cpufreq_ops tegra194_cpufreq_ops = { + .read_counters = tegra194_read_counters, + .get_cpu_cluster_id = tegra194_get_cpu_cluster_id, + .get_cpu_ndiv = tegra194_get_cpu_ndiv, + .set_cpu_ndiv = tegra194_set_cpu_ndiv, +}; + +static const struct tegra_cpufreq_soc tegra194_cpufreq_soc = { + .ops = &tegra194_cpufreq_ops, + .maxcpus_per_cluster = 2, + .num_clusters = 4, + .refclk_delta_min = 16000, +}; + +static void tegra194_cpufreq_free_resources(void) +{ + destroy_workqueue(read_counters_wq); +} + +static struct cpufreq_frequency_table * +tegra_cpufreq_bpmp_read_lut(struct platform_device *pdev, struct tegra_bpmp *bpmp, + unsigned int cluster_id) +{ + struct cpufreq_frequency_table *freq_table; + struct mrq_cpu_ndiv_limits_response resp; + unsigned int num_freqs, ndiv, delta_ndiv; + struct mrq_cpu_ndiv_limits_request req; + struct tegra_bpmp_message msg; + u16 freq_table_step_size; + int err, index; + + memset(&req, 0, sizeof(req)); + req.cluster_id = cluster_id; + + memset(&msg, 0, sizeof(msg)); + msg.mrq = MRQ_CPU_NDIV_LIMITS; + msg.tx.data = &req; + msg.tx.size = sizeof(req); + msg.rx.data = &resp; + msg.rx.size = sizeof(resp); + + err = tegra_bpmp_transfer(bpmp, &msg); + if (err) + return ERR_PTR(err); + if (msg.rx.ret == -BPMP_EINVAL) { + /* Cluster not available */ + return NULL; + } + if (msg.rx.ret) + return ERR_PTR(-EINVAL); + + /* + * Make sure frequency table step is a multiple of mdiv to match + * vhint table granularity. + */ + freq_table_step_size = resp.mdiv * + DIV_ROUND_UP(CPUFREQ_TBL_STEP_HZ, resp.ref_clk_hz); + + dev_dbg(&pdev->dev, "cluster %d: frequency table step size: %d\n", + cluster_id, freq_table_step_size); + + delta_ndiv = resp.ndiv_max - resp.ndiv_min; + + if (unlikely(delta_ndiv == 0)) { + num_freqs = 1; + } else { + /* We store both ndiv_min and ndiv_max hence the +1 */ + num_freqs = delta_ndiv / freq_table_step_size + 1; + } + + num_freqs += (delta_ndiv % freq_table_step_size) ? 1 : 0; + + freq_table = devm_kcalloc(&pdev->dev, num_freqs + 1, + sizeof(*freq_table), GFP_KERNEL); + if (!freq_table) + return ERR_PTR(-ENOMEM); + + for (index = 0, ndiv = resp.ndiv_min; + ndiv < resp.ndiv_max; + index++, ndiv += freq_table_step_size) { + freq_table[index].driver_data = ndiv; + freq_table[index].frequency = map_ndiv_to_freq(&resp, ndiv); + } + + freq_table[index].driver_data = resp.ndiv_max; + freq_table[index++].frequency = map_ndiv_to_freq(&resp, resp.ndiv_max); + freq_table[index].frequency = CPUFREQ_TABLE_END; + + return freq_table; +} + +static int tegra194_cpufreq_store_physids(unsigned int cpu, struct tegra194_cpufreq_data *data) +{ + int num_cpus = data->soc->maxcpus_per_cluster * data->soc->num_clusters; + u32 cpuid, clusterid; + u64 mpidr_id; + + if (cpu > (num_cpus - 1)) { + pr_err("cpufreq: wrong num of cpus or clusters in soc data\n"); + return -EINVAL; + } + + data->soc->ops->get_cpu_cluster_id(cpu, &cpuid, &clusterid); + + mpidr_id = (clusterid * data->soc->maxcpus_per_cluster) + cpuid; + + data->cpu_data[cpu].cpuid = cpuid; + data->cpu_data[cpu].clusterid = clusterid; + data->cpu_data[cpu].freq_core_reg = SCRATCH_FREQ_CORE_REG(data, mpidr_id); + + return 0; +} + +static int tegra194_cpufreq_probe(struct platform_device *pdev) +{ + const struct tegra_cpufreq_soc *soc; + struct tegra194_cpufreq_data *data; + struct tegra_bpmp *bpmp; + struct device *cpu_dev; + int err, i; + u32 cpu; + + data = devm_kzalloc(&pdev->dev, sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + soc = of_device_get_match_data(&pdev->dev); + + if (soc->ops && soc->maxcpus_per_cluster && soc->num_clusters && soc->refclk_delta_min) { + data->soc = soc; + } else { + dev_err(&pdev->dev, "soc data missing\n"); + return -EINVAL; + } + + data->bpmp_luts = devm_kcalloc(&pdev->dev, data->soc->num_clusters, + sizeof(*data->bpmp_luts), GFP_KERNEL); + if (!data->bpmp_luts) + return -ENOMEM; + + if (soc->actmon_cntr_base) { + /* mmio registers are used for frequency request and re-construction */ + data->regs = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(data->regs)) + return PTR_ERR(data->regs); + } + + data->cpu_data = devm_kcalloc(&pdev->dev, data->soc->num_clusters * + data->soc->maxcpus_per_cluster, + sizeof(*data->cpu_data), GFP_KERNEL); + if (!data->cpu_data) + return -ENOMEM; + + platform_set_drvdata(pdev, data); + + bpmp = tegra_bpmp_get(&pdev->dev); + if (IS_ERR(bpmp)) + return PTR_ERR(bpmp); + + read_counters_wq = alloc_workqueue("read_counters_wq", + __WQ_LEGACY | WQ_PERCPU, 1); + if (!read_counters_wq) { + dev_err(&pdev->dev, "fail to create_workqueue\n"); + err = -EINVAL; + goto put_bpmp; + } + + for (i = 0; i < data->soc->num_clusters; i++) { + data->bpmp_luts[i] = tegra_cpufreq_bpmp_read_lut(pdev, bpmp, i); + if (IS_ERR(data->bpmp_luts[i])) { + err = PTR_ERR(data->bpmp_luts[i]); + goto err_free_res; + } + } + + for_each_possible_cpu(cpu) { + err = tegra194_cpufreq_store_physids(cpu, data); + if (err) + goto err_free_res; + } + + tegra194_cpufreq_driver.driver_data = data; + + /* Check for optional OPPv2 and interconnect paths on CPU0 to enable ICC scaling */ + cpu_dev = get_cpu_device(0); + if (!cpu_dev) { + err = -EPROBE_DEFER; + goto err_free_res; + } + + if (dev_pm_opp_of_get_opp_desc_node(cpu_dev)) { + err = dev_pm_opp_of_find_icc_paths(cpu_dev, NULL); + if (!err) + data->icc_dram_bw_scaling = true; + } + + err = cpufreq_register_driver(&tegra194_cpufreq_driver); + if (!err) + goto put_bpmp; + +err_free_res: + tegra194_cpufreq_free_resources(); +put_bpmp: + tegra_bpmp_put(bpmp); + return err; +} + +static void tegra194_cpufreq_remove(struct platform_device *pdev) +{ + cpufreq_unregister_driver(&tegra194_cpufreq_driver); + tegra194_cpufreq_free_resources(); +} + +static const struct of_device_id tegra194_cpufreq_of_match[] = { + { .compatible = "nvidia,tegra194-ccplex", .data = &tegra194_cpufreq_soc }, + { .compatible = "nvidia,tegra234-ccplex-cluster", .data = &tegra234_cpufreq_soc }, + { .compatible = "nvidia,tegra239-ccplex-cluster", .data = &tegra239_cpufreq_soc }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, tegra194_cpufreq_of_match); + +static struct platform_driver tegra194_ccplex_driver = { + .driver = { + .name = "tegra194-cpufreq", + .of_match_table = tegra194_cpufreq_of_match, + }, + .probe = tegra194_cpufreq_probe, + .remove = tegra194_cpufreq_remove, +}; +module_platform_driver(tegra194_ccplex_driver); + +MODULE_AUTHOR("Mikko Perttunen <mperttunen@nvidia.com>"); +MODULE_AUTHOR("Sumit Gupta <sumitg@nvidia.com>"); +MODULE_DESCRIPTION("NVIDIA Tegra194 cpufreq driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/cpufreq/tegra20-cpufreq.c b/drivers/cpufreq/tegra20-cpufreq.c new file mode 100644 index 000000000000..a573186704a5 --- /dev/null +++ b/drivers/cpufreq/tegra20-cpufreq.c @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2010 Google, Inc. + * + * Author: + * Colin Cross <ccross@google.com> + * Based on arch/arm/plat-omap/cpu-omap.c, (C) 2005 Nokia Corporation + */ + +#include <linux/bits.h> +#include <linux/cpu.h> +#include <linux/err.h> +#include <linux/init.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/platform_device.h> +#include <linux/pm_opp.h> +#include <linux/types.h> + +#include <soc/tegra/common.h> +#include <soc/tegra/fuse.h> + +static bool cpu0_node_has_opp_v2_prop(void) +{ + struct device_node *np = of_cpu_device_node_get(0); + bool ret = false; + + if (of_property_present(np, "operating-points-v2")) + ret = true; + + of_node_put(np); + return ret; +} + +static void tegra20_cpufreq_put_supported_hw(void *opp_token) +{ + dev_pm_opp_put_supported_hw((unsigned long) opp_token); +} + +static void tegra20_cpufreq_dt_unregister(void *cpufreq_dt) +{ + platform_device_unregister(cpufreq_dt); +} + +static int tegra20_cpufreq_probe(struct platform_device *pdev) +{ + struct platform_device *cpufreq_dt; + struct device *cpu_dev; + u32 versions[2]; + int err; + + if (!cpu0_node_has_opp_v2_prop()) { + dev_err(&pdev->dev, "operating points not found\n"); + dev_err(&pdev->dev, "please update your device tree\n"); + return -ENODEV; + } + + if (of_machine_is_compatible("nvidia,tegra20")) { + versions[0] = BIT(tegra_sku_info.cpu_process_id); + versions[1] = BIT(tegra_sku_info.soc_speedo_id); + } else { + versions[0] = BIT(tegra_sku_info.cpu_process_id); + versions[1] = BIT(tegra_sku_info.cpu_speedo_id); + } + + dev_info(&pdev->dev, "hardware version 0x%x 0x%x\n", + versions[0], versions[1]); + + cpu_dev = get_cpu_device(0); + if (WARN_ON(!cpu_dev)) + return -ENODEV; + + err = dev_pm_opp_set_supported_hw(cpu_dev, versions, 2); + if (err < 0) { + dev_err(&pdev->dev, "failed to set supported hw: %d\n", err); + return err; + } + + err = devm_add_action_or_reset(&pdev->dev, + tegra20_cpufreq_put_supported_hw, + (void *)((unsigned long) err)); + if (err) + return err; + + cpufreq_dt = platform_device_register_simple("cpufreq-dt", -1, NULL, 0); + err = PTR_ERR_OR_ZERO(cpufreq_dt); + if (err) { + dev_err(&pdev->dev, + "failed to create cpufreq-dt device: %d\n", err); + return err; + } + + err = devm_add_action_or_reset(&pdev->dev, + tegra20_cpufreq_dt_unregister, + cpufreq_dt); + if (err) + return err; + + return 0; +} + +static struct platform_driver tegra20_cpufreq_driver = { + .probe = tegra20_cpufreq_probe, + .driver = { + .name = "tegra20-cpufreq", + }, +}; +module_platform_driver(tegra20_cpufreq_driver); + +MODULE_ALIAS("platform:tegra20-cpufreq"); +MODULE_AUTHOR("Colin Cross <ccross@android.com>"); +MODULE_DESCRIPTION("NVIDIA Tegra20 cpufreq driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/cpufreq/ti-cpufreq.c b/drivers/cpufreq/ti-cpufreq.c new file mode 100644 index 000000000000..6ee76f5fe9c5 --- /dev/null +++ b/drivers/cpufreq/ti-cpufreq.c @@ -0,0 +1,586 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * TI CPUFreq/OPP hw-supported driver + * + * Copyright (C) 2016-2017 Texas Instruments, Inc. + * Dave Gerlach <d-gerlach@ti.com> + */ + +#include <linux/cpu.h> +#include <linux/io.h> +#include <linux/mfd/syscon.h> +#include <linux/module.h> +#include <linux/init.h> +#include <linux/of.h> +#include <linux/platform_device.h> +#include <linux/pm_opp.h> +#include <linux/regmap.h> +#include <linux/slab.h> +#include <linux/sys_soc.h> + +#define REVISION_MASK 0xF +#define REVISION_SHIFT 28 + +#define AM33XX_800M_ARM_MPU_MAX_FREQ 0x1E2F +#define AM43XX_600M_ARM_MPU_MAX_FREQ 0xFFA + +#define DRA7_EFUSE_HAS_OD_MPU_OPP 11 +#define DRA7_EFUSE_HAS_HIGH_MPU_OPP 15 +#define DRA76_EFUSE_HAS_PLUS_MPU_OPP 18 +#define DRA7_EFUSE_HAS_ALL_MPU_OPP 23 +#define DRA76_EFUSE_HAS_ALL_MPU_OPP 24 + +#define DRA7_EFUSE_NOM_MPU_OPP BIT(0) +#define DRA7_EFUSE_OD_MPU_OPP BIT(1) +#define DRA7_EFUSE_HIGH_MPU_OPP BIT(2) +#define DRA76_EFUSE_PLUS_MPU_OPP BIT(3) + +#define OMAP3_CONTROL_DEVICE_STATUS 0x4800244C +#define OMAP3_CONTROL_IDCODE 0x4830A204 +#define OMAP34xx_ProdID_SKUID 0x4830A20C +#define OMAP3_SYSCON_BASE (0x48000000 + 0x2000 + 0x270) + +#define AM625_EFUSE_K_MPU_OPP 11 +#define AM625_EFUSE_S_MPU_OPP 19 +#define AM625_EFUSE_T_MPU_OPP 20 + +#define AM625_SUPPORT_K_MPU_OPP BIT(0) +#define AM625_SUPPORT_S_MPU_OPP BIT(1) +#define AM625_SUPPORT_T_MPU_OPP BIT(2) + +enum { + AM62A7_EFUSE_M_MPU_OPP = 13, + AM62A7_EFUSE_N_MPU_OPP, + AM62A7_EFUSE_O_MPU_OPP, + AM62A7_EFUSE_P_MPU_OPP, + AM62A7_EFUSE_Q_MPU_OPP, + AM62A7_EFUSE_R_MPU_OPP, + AM62A7_EFUSE_S_MPU_OPP, + /* + * The V, U, and T speed grade numbering is out of order + * to align with the AM625 more uniformly. I promise I know + * my ABCs ;) + */ + AM62A7_EFUSE_V_MPU_OPP, + AM62A7_EFUSE_U_MPU_OPP, + AM62A7_EFUSE_T_MPU_OPP, +}; + +#define AM62A7_SUPPORT_N_MPU_OPP BIT(0) +#define AM62A7_SUPPORT_R_MPU_OPP BIT(1) +#define AM62A7_SUPPORT_V_MPU_OPP BIT(2) + +#define AM62P5_EFUSE_O_MPU_OPP 15 +#define AM62P5_EFUSE_S_MPU_OPP 19 +#define AM62P5_EFUSE_T_MPU_OPP 20 +#define AM62P5_EFUSE_U_MPU_OPP 21 +#define AM62P5_EFUSE_V_MPU_OPP 22 + +#define AM62P5_SUPPORT_O_MPU_OPP BIT(0) +#define AM62P5_SUPPORT_U_MPU_OPP BIT(2) + +#define VERSION_COUNT 2 + +struct ti_cpufreq_data; + +struct ti_cpufreq_soc_data { + const char * const *reg_names; + unsigned long (*efuse_xlate)(struct ti_cpufreq_data *opp_data, + unsigned long efuse); + unsigned long efuse_fallback; + unsigned long efuse_offset; + unsigned long efuse_mask; + unsigned long efuse_shift; + unsigned long rev_offset; + bool multi_regulator; +/* Backward compatibility hack: Might have missing syscon */ +#define TI_QUIRK_SYSCON_MAY_BE_MISSING 0x1 +/* Backward compatibility hack: new syscon size is 1 register wide */ +#define TI_QUIRK_SYSCON_IS_SINGLE_REG 0x2 + u8 quirks; +}; + +struct ti_cpufreq_data { + struct device *cpu_dev; + struct device_node *opp_node; + struct regmap *syscon; + const struct ti_cpufreq_soc_data *soc_data; +}; + +static unsigned long amx3_efuse_xlate(struct ti_cpufreq_data *opp_data, + unsigned long efuse) +{ + if (!efuse) + efuse = opp_data->soc_data->efuse_fallback; + /* AM335x and AM437x use "OPP disable" bits, so invert */ + return ~efuse; +} + +static unsigned long dra7_efuse_xlate(struct ti_cpufreq_data *opp_data, + unsigned long efuse) +{ + unsigned long calculated_efuse = DRA7_EFUSE_NOM_MPU_OPP; + + /* + * The efuse on dra7 and am57 parts contains a specific + * value indicating the highest available OPP. + */ + + switch (efuse) { + case DRA76_EFUSE_HAS_PLUS_MPU_OPP: + case DRA76_EFUSE_HAS_ALL_MPU_OPP: + calculated_efuse |= DRA76_EFUSE_PLUS_MPU_OPP; + fallthrough; + case DRA7_EFUSE_HAS_ALL_MPU_OPP: + case DRA7_EFUSE_HAS_HIGH_MPU_OPP: + calculated_efuse |= DRA7_EFUSE_HIGH_MPU_OPP; + fallthrough; + case DRA7_EFUSE_HAS_OD_MPU_OPP: + calculated_efuse |= DRA7_EFUSE_OD_MPU_OPP; + } + + return calculated_efuse; +} + +static unsigned long omap3_efuse_xlate(struct ti_cpufreq_data *opp_data, + unsigned long efuse) +{ + /* OPP enable bit ("Speed Binned") */ + return BIT(efuse); +} + +static unsigned long am62p5_efuse_xlate(struct ti_cpufreq_data *opp_data, + unsigned long efuse) +{ + unsigned long calculated_efuse = AM62P5_SUPPORT_O_MPU_OPP; + + switch (efuse) { + case AM62P5_EFUSE_V_MPU_OPP: + case AM62P5_EFUSE_U_MPU_OPP: + case AM62P5_EFUSE_T_MPU_OPP: + case AM62P5_EFUSE_S_MPU_OPP: + calculated_efuse |= AM62P5_SUPPORT_U_MPU_OPP; + fallthrough; + case AM62P5_EFUSE_O_MPU_OPP: + calculated_efuse |= AM62P5_SUPPORT_O_MPU_OPP; + } + + return calculated_efuse; +} + +static unsigned long am62a7_efuse_xlate(struct ti_cpufreq_data *opp_data, + unsigned long efuse) +{ + unsigned long calculated_efuse = AM62A7_SUPPORT_N_MPU_OPP; + + switch (efuse) { + case AM62A7_EFUSE_V_MPU_OPP: + case AM62A7_EFUSE_U_MPU_OPP: + case AM62A7_EFUSE_T_MPU_OPP: + case AM62A7_EFUSE_S_MPU_OPP: + calculated_efuse |= AM62A7_SUPPORT_V_MPU_OPP; + fallthrough; + case AM62A7_EFUSE_R_MPU_OPP: + case AM62A7_EFUSE_Q_MPU_OPP: + case AM62A7_EFUSE_P_MPU_OPP: + case AM62A7_EFUSE_O_MPU_OPP: + calculated_efuse |= AM62A7_SUPPORT_R_MPU_OPP; + fallthrough; + case AM62A7_EFUSE_N_MPU_OPP: + case AM62A7_EFUSE_M_MPU_OPP: + calculated_efuse |= AM62A7_SUPPORT_N_MPU_OPP; + } + + return calculated_efuse; +} + +static unsigned long am625_efuse_xlate(struct ti_cpufreq_data *opp_data, + unsigned long efuse) +{ + unsigned long calculated_efuse = AM625_SUPPORT_K_MPU_OPP; + + switch (efuse) { + case AM625_EFUSE_T_MPU_OPP: + calculated_efuse |= AM625_SUPPORT_T_MPU_OPP; + fallthrough; + case AM625_EFUSE_S_MPU_OPP: + calculated_efuse |= AM625_SUPPORT_S_MPU_OPP; + fallthrough; + case AM625_EFUSE_K_MPU_OPP: + calculated_efuse |= AM625_SUPPORT_K_MPU_OPP; + } + + return calculated_efuse; +} + +static struct ti_cpufreq_soc_data am3x_soc_data = { + .efuse_xlate = amx3_efuse_xlate, + .efuse_fallback = AM33XX_800M_ARM_MPU_MAX_FREQ, + .efuse_offset = 0x07fc, + .efuse_mask = 0x1fff, + .rev_offset = 0x600, + .multi_regulator = false, +}; + +static struct ti_cpufreq_soc_data am4x_soc_data = { + .efuse_xlate = amx3_efuse_xlate, + .efuse_fallback = AM43XX_600M_ARM_MPU_MAX_FREQ, + .efuse_offset = 0x0610, + .efuse_mask = 0x3f, + .rev_offset = 0x600, + .multi_regulator = false, +}; + +static struct ti_cpufreq_soc_data dra7_soc_data = { + .efuse_xlate = dra7_efuse_xlate, + .efuse_offset = 0x020c, + .efuse_mask = 0xf80000, + .efuse_shift = 19, + .rev_offset = 0x204, + .multi_regulator = true, +}; + +/* + * OMAP35x TRM (SPRUF98K): + * CONTROL_IDCODE (0x4830 A204) describes Silicon revisions. + * Control OMAP Status Register 15:0 (Address 0x4800 244C) + * to separate between omap3503, omap3515, omap3525, omap3530 + * and feature presence. + * There are encodings for versions limited to 400/266MHz + * but we ignore. + * Not clear if this also holds for omap34xx. + * some eFuse values e.g. CONTROL_FUSE_OPP1_VDD1 + * are stored in the SYSCON register range + * Register 0x4830A20C [ProdID.SKUID] [0:3] + * 0x0 for normal 600/430MHz device. + * 0x8 for 720/520MHz device. + * Not clear what omap34xx value is. + */ + +static struct ti_cpufreq_soc_data omap34xx_soc_data = { + .efuse_xlate = omap3_efuse_xlate, + .efuse_offset = OMAP34xx_ProdID_SKUID - OMAP3_SYSCON_BASE, + .efuse_shift = 3, + .efuse_mask = BIT(3), + .rev_offset = OMAP3_CONTROL_IDCODE - OMAP3_SYSCON_BASE, + .multi_regulator = false, + .quirks = TI_QUIRK_SYSCON_MAY_BE_MISSING, +}; + +/* + * AM/DM37x TRM (SPRUGN4M) + * CONTROL_IDCODE (0x4830 A204) describes Silicon revisions. + * Control Device Status Register 15:0 (Address 0x4800 244C) + * to separate between am3703, am3715, dm3725, dm3730 + * and feature presence. + * Speed Binned = Bit 9 + * 0 800/600 MHz + * 1 1000/800 MHz + * some eFuse values e.g. CONTROL_FUSE_OPP 1G_VDD1 + * are stored in the SYSCON register range. + * There is no 0x4830A20C [ProdID.SKUID] register (exists but + * seems to always read as 0). + */ + +static const char * const omap3_reg_names[] = {"cpu0", "vbb", NULL}; + +static struct ti_cpufreq_soc_data omap36xx_soc_data = { + .reg_names = omap3_reg_names, + .efuse_xlate = omap3_efuse_xlate, + .efuse_offset = OMAP3_CONTROL_DEVICE_STATUS - OMAP3_SYSCON_BASE, + .efuse_shift = 9, + .efuse_mask = BIT(9), + .rev_offset = OMAP3_CONTROL_IDCODE - OMAP3_SYSCON_BASE, + .multi_regulator = true, + .quirks = TI_QUIRK_SYSCON_MAY_BE_MISSING, +}; + +/* + * AM3517 is quite similar to AM/DM37x except that it has no + * high speed grade eFuse and no abb ldo + */ + +static struct ti_cpufreq_soc_data am3517_soc_data = { + .efuse_xlate = omap3_efuse_xlate, + .efuse_offset = OMAP3_CONTROL_DEVICE_STATUS - OMAP3_SYSCON_BASE, + .efuse_shift = 0, + .efuse_mask = 0, + .rev_offset = OMAP3_CONTROL_IDCODE - OMAP3_SYSCON_BASE, + .multi_regulator = false, + .quirks = TI_QUIRK_SYSCON_MAY_BE_MISSING, +}; + +static const struct soc_device_attribute k3_cpufreq_soc[] = { + { .family = "AM62X", }, + { .family = "AM62AX", }, + { .family = "AM62PX", }, + { .family = "AM62DX", }, + { /* sentinel */ } +}; + +static struct ti_cpufreq_soc_data am625_soc_data = { + .efuse_xlate = am625_efuse_xlate, + .efuse_offset = 0x0018, + .efuse_mask = 0x07c0, + .efuse_shift = 0x6, + .multi_regulator = false, + .quirks = TI_QUIRK_SYSCON_IS_SINGLE_REG, +}; + +static struct ti_cpufreq_soc_data am62a7_soc_data = { + .efuse_xlate = am62a7_efuse_xlate, + .efuse_offset = 0x0, + .efuse_mask = 0x07c0, + .efuse_shift = 0x6, + .multi_regulator = false, +}; + +static struct ti_cpufreq_soc_data am62p5_soc_data = { + .efuse_xlate = am62p5_efuse_xlate, + .efuse_offset = 0x0, + .efuse_mask = 0x07c0, + .efuse_shift = 0x6, + .multi_regulator = false, +}; + +/** + * ti_cpufreq_get_efuse() - Parse and return efuse value present on SoC + * @opp_data: pointer to ti_cpufreq_data context + * @efuse_value: Set to the value parsed from efuse + * + * Returns error code if efuse not read properly. + */ +static int ti_cpufreq_get_efuse(struct ti_cpufreq_data *opp_data, + u32 *efuse_value) +{ + struct device *dev = opp_data->cpu_dev; + u32 efuse; + int ret; + + ret = regmap_read(opp_data->syscon, opp_data->soc_data->efuse_offset, + &efuse); + + if (opp_data->soc_data->quirks & TI_QUIRK_SYSCON_IS_SINGLE_REG && ret == -EIO) + ret = regmap_read(opp_data->syscon, 0x0, &efuse); + + if (opp_data->soc_data->quirks & TI_QUIRK_SYSCON_MAY_BE_MISSING && ret == -EIO) { + /* not a syscon register! */ + void __iomem *regs = ioremap(OMAP3_SYSCON_BASE + + opp_data->soc_data->efuse_offset, 4); + + if (!regs) + return -ENOMEM; + efuse = readl(regs); + iounmap(regs); + } + else if (ret) { + dev_err(dev, + "Failed to read the efuse value from syscon: %d\n", + ret); + return ret; + } + + efuse = (efuse & opp_data->soc_data->efuse_mask); + efuse >>= opp_data->soc_data->efuse_shift; + + *efuse_value = opp_data->soc_data->efuse_xlate(opp_data, efuse); + + return 0; +} + +/** + * ti_cpufreq_get_rev() - Parse and return rev value present on SoC + * @opp_data: pointer to ti_cpufreq_data context + * @revision_value: Set to the value parsed from revision register + * + * Returns error code if revision not read properly. + */ +static int ti_cpufreq_get_rev(struct ti_cpufreq_data *opp_data, + u32 *revision_value) +{ + struct device *dev = opp_data->cpu_dev; + u32 revision; + int ret; + if (soc_device_match(k3_cpufreq_soc)) { + /* + * Since the SR is 1.0, hard code the revision_value as + * 0x1 here. This way we avoid re using the same register + * that is giving us required information inside socinfo + * anyway. + */ + *revision_value = 0x1; + goto done; + } + + ret = regmap_read(opp_data->syscon, opp_data->soc_data->rev_offset, + &revision); + if (opp_data->soc_data->quirks & TI_QUIRK_SYSCON_MAY_BE_MISSING && ret == -EIO) { + /* not a syscon register! */ + void __iomem *regs = ioremap(OMAP3_SYSCON_BASE + + opp_data->soc_data->rev_offset, 4); + + if (!regs) + return -ENOMEM; + revision = readl(regs); + iounmap(regs); + } + else if (ret) { + dev_err(dev, + "Failed to read the revision number from syscon: %d\n", + ret); + return ret; + } + + *revision_value = BIT((revision >> REVISION_SHIFT) & REVISION_MASK); + +done: + return 0; +} + +static int ti_cpufreq_setup_syscon_register(struct ti_cpufreq_data *opp_data) +{ + struct device *dev = opp_data->cpu_dev; + struct device_node *np = opp_data->opp_node; + + opp_data->syscon = syscon_regmap_lookup_by_phandle(np, + "syscon"); + if (IS_ERR(opp_data->syscon)) { + dev_err(dev, + "\"syscon\" is missing, cannot use OPPv2 table.\n"); + return PTR_ERR(opp_data->syscon); + } + + return 0; +} + +static const struct of_device_id ti_cpufreq_of_match[] __maybe_unused = { + { .compatible = "ti,am33xx", .data = &am3x_soc_data, }, + { .compatible = "ti,am3517", .data = &am3517_soc_data, }, + { .compatible = "ti,am43", .data = &am4x_soc_data, }, + { .compatible = "ti,dra7", .data = &dra7_soc_data }, + { .compatible = "ti,omap34xx", .data = &omap34xx_soc_data, }, + { .compatible = "ti,omap36xx", .data = &omap36xx_soc_data, }, + { .compatible = "ti,am625", .data = &am625_soc_data, }, + { .compatible = "ti,am62a7", .data = &am62a7_soc_data, }, + { .compatible = "ti,am62d2", .data = &am62a7_soc_data, }, + { .compatible = "ti,am62p5", .data = &am62p5_soc_data, }, + /* legacy */ + { .compatible = "ti,omap3430", .data = &omap34xx_soc_data, }, + { .compatible = "ti,omap3630", .data = &omap36xx_soc_data, }, + {}, +}; + +static const struct of_device_id *ti_cpufreq_match_node(void) +{ + struct device_node *np __free(device_node) = of_find_node_by_path("/"); + const struct of_device_id *match; + + match = of_match_node(ti_cpufreq_of_match, np); + + return match; +} + +static int ti_cpufreq_probe(struct platform_device *pdev) +{ + u32 version[VERSION_COUNT]; + const struct of_device_id *match; + struct ti_cpufreq_data *opp_data; + const char * const default_reg_names[] = {"vdd", "vbb", NULL}; + int ret; + struct dev_pm_opp_config config = { + .supported_hw = version, + .supported_hw_count = ARRAY_SIZE(version), + }; + + match = dev_get_platdata(&pdev->dev); + if (!match) + return -ENODEV; + + opp_data = devm_kzalloc(&pdev->dev, sizeof(*opp_data), GFP_KERNEL); + if (!opp_data) + return -ENOMEM; + + opp_data->soc_data = match->data; + + opp_data->cpu_dev = get_cpu_device(0); + if (!opp_data->cpu_dev) { + pr_err("%s: Failed to get device for CPU0\n", __func__); + return -ENODEV; + } + + opp_data->opp_node = dev_pm_opp_of_get_opp_desc_node(opp_data->cpu_dev); + if (!opp_data->opp_node) { + dev_info(opp_data->cpu_dev, + "OPP-v2 not supported, cpufreq-dt will attempt to use legacy tables.\n"); + goto register_cpufreq_dt; + } + + ret = ti_cpufreq_setup_syscon_register(opp_data); + if (ret) + goto fail_put_node; + + /* + * OPPs determine whether or not they are supported based on + * two metrics: + * 0 - SoC Revision + * 1 - eFuse value + */ + ret = ti_cpufreq_get_rev(opp_data, &version[0]); + if (ret) + goto fail_put_node; + + ret = ti_cpufreq_get_efuse(opp_data, &version[1]); + if (ret) + goto fail_put_node; + + if (opp_data->soc_data->multi_regulator) { + if (opp_data->soc_data->reg_names) + config.regulator_names = opp_data->soc_data->reg_names; + else + config.regulator_names = default_reg_names; + } + + ret = dev_pm_opp_set_config(opp_data->cpu_dev, &config); + if (ret < 0) { + dev_err_probe(opp_data->cpu_dev, ret, "Failed to set OPP config\n"); + goto fail_put_node; + } + + of_node_put(opp_data->opp_node); + +register_cpufreq_dt: + platform_device_register_simple("cpufreq-dt", -1, NULL, 0); + + return 0; + +fail_put_node: + of_node_put(opp_data->opp_node); + + return ret; +} + +static int __init ti_cpufreq_init(void) +{ + const struct of_device_id *match; + + /* Check to ensure we are on a compatible platform */ + match = ti_cpufreq_match_node(); + if (match) + platform_device_register_data(NULL, "ti-cpufreq", -1, match, + sizeof(*match)); + + return 0; +} +module_init(ti_cpufreq_init); + +static struct platform_driver ti_cpufreq_driver = { + .probe = ti_cpufreq_probe, + .driver = { + .name = "ti-cpufreq", + }, +}; +builtin_platform_driver(ti_cpufreq_driver); + +MODULE_DESCRIPTION("TI CPUFreq/OPP hw-supported driver"); +MODULE_AUTHOR("Dave Gerlach <d-gerlach@ti.com>"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/cpufreq/unicore2-cpufreq.c b/drivers/cpufreq/unicore2-cpufreq.c deleted file mode 100644 index 12fc904d7dab..000000000000 --- a/drivers/cpufreq/unicore2-cpufreq.c +++ /dev/null @@ -1,92 +0,0 @@ -/* - * clock scaling for the UniCore-II - * - * Code specific to PKUnity SoC and UniCore ISA - * - * Maintained by GUAN Xue-tao <gxt@mprc.pku.edu.cn> - * Copyright (C) 2001-2010 Guan Xuetao - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - */ - -#include <linux/kernel.h> -#include <linux/types.h> -#include <linux/init.h> -#include <linux/clk.h> -#include <linux/cpufreq.h> - -#include <mach/hardware.h> - -static struct cpufreq_driver ucv2_driver; - -/* make sure that only the "userspace" governor is run - * -- anything else wouldn't make sense on this platform, anyway. - */ -int ucv2_verify_speed(struct cpufreq_policy *policy) -{ - if (policy->cpu) - return -EINVAL; - - cpufreq_verify_within_limits(policy, - policy->cpuinfo.min_freq, policy->cpuinfo.max_freq); - - return 0; -} - -static unsigned int ucv2_getspeed(unsigned int cpu) -{ - struct clk *mclk = clk_get(NULL, "MAIN_CLK"); - - if (cpu) - return 0; - return clk_get_rate(mclk)/1000; -} - -static int ucv2_target(struct cpufreq_policy *policy, - unsigned int target_freq, - unsigned int relation) -{ - unsigned int cur = ucv2_getspeed(0); - struct cpufreq_freqs freqs; - struct clk *mclk = clk_get(NULL, "MAIN_CLK"); - - cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE); - - if (!clk_set_rate(mclk, target_freq * 1000)) { - freqs.old = cur; - freqs.new = target_freq; - } - - cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE); - - return 0; -} - -static int __init ucv2_cpu_init(struct cpufreq_policy *policy) -{ - if (policy->cpu != 0) - return -EINVAL; - policy->cur = ucv2_getspeed(0); - policy->min = policy->cpuinfo.min_freq = 250000; - policy->max = policy->cpuinfo.max_freq = 1000000; - policy->cpuinfo.transition_latency = CPUFREQ_ETERNAL; - return 0; -} - -static struct cpufreq_driver ucv2_driver = { - .flags = CPUFREQ_STICKY, - .verify = ucv2_verify_speed, - .target = ucv2_target, - .get = ucv2_getspeed, - .init = ucv2_cpu_init, - .name = "UniCore-II", -}; - -static int __init ucv2_cpufreq_init(void) -{ - return cpufreq_register_driver(&ucv2_driver); -} - -arch_initcall(ucv2_cpufreq_init); diff --git a/drivers/cpufreq/vexpress-spc-cpufreq.c b/drivers/cpufreq/vexpress-spc-cpufreq.c new file mode 100644 index 000000000000..65fea47b82e6 --- /dev/null +++ b/drivers/cpufreq/vexpress-spc-cpufreq.c @@ -0,0 +1,575 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Versatile Express SPC CPUFreq Interface driver + * + * Copyright (C) 2013 - 2019 ARM Ltd. + * Sudeep Holla <sudeep.holla@arm.com> + * + * Copyright (C) 2013 Linaro. + * Viresh Kumar <viresh.kumar@linaro.org> + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/clk.h> +#include <linux/cpu.h> +#include <linux/cpufreq.h> +#include <linux/cpumask.h> +#include <linux/device.h> +#include <linux/module.h> +#include <linux/mutex.h> +#include <linux/platform_device.h> +#include <linux/pm_opp.h> +#include <linux/slab.h> +#include <linux/topology.h> +#include <linux/types.h> + +/* Currently we support only two clusters */ +#define A15_CLUSTER 0 +#define A7_CLUSTER 1 +#define MAX_CLUSTERS 2 + +#ifdef CONFIG_BL_SWITCHER +#include <asm/bL_switcher.h> +static bool bL_switching_enabled; +#define is_bL_switching_enabled() bL_switching_enabled +#define set_switching_enabled(x) (bL_switching_enabled = (x)) +#else +#define is_bL_switching_enabled() false +#define set_switching_enabled(x) do { } while (0) +#define bL_switch_request(...) do { } while (0) +#define bL_switcher_put_enabled() do { } while (0) +#define bL_switcher_get_enabled() do { } while (0) +#endif + +#define ACTUAL_FREQ(cluster, freq) ((cluster == A7_CLUSTER) ? freq << 1 : freq) +#define VIRT_FREQ(cluster, freq) ((cluster == A7_CLUSTER) ? freq >> 1 : freq) + +static struct clk *clk[MAX_CLUSTERS]; +static struct cpufreq_frequency_table *freq_table[MAX_CLUSTERS + 1]; +static atomic_t cluster_usage[MAX_CLUSTERS + 1]; + +static unsigned int clk_big_min; /* (Big) clock frequencies */ +static unsigned int clk_little_max; /* Maximum clock frequency (Little) */ + +static DEFINE_PER_CPU(unsigned int, physical_cluster); +static DEFINE_PER_CPU(unsigned int, cpu_last_req_freq); + +static struct mutex cluster_lock[MAX_CLUSTERS]; + +static inline int raw_cpu_to_cluster(int cpu) +{ + return topology_physical_package_id(cpu); +} + +static inline int cpu_to_cluster(int cpu) +{ + return is_bL_switching_enabled() ? + MAX_CLUSTERS : raw_cpu_to_cluster(cpu); +} + +static unsigned int find_cluster_maxfreq(int cluster) +{ + int j; + u32 max_freq = 0, cpu_freq; + + for_each_online_cpu(j) { + cpu_freq = per_cpu(cpu_last_req_freq, j); + + if (cluster == per_cpu(physical_cluster, j) && + max_freq < cpu_freq) + max_freq = cpu_freq; + } + + return max_freq; +} + +static unsigned int clk_get_cpu_rate(unsigned int cpu) +{ + u32 cur_cluster = per_cpu(physical_cluster, cpu); + u32 rate = clk_get_rate(clk[cur_cluster]) / 1000; + + /* For switcher we use virtual A7 clock rates */ + if (is_bL_switching_enabled()) + rate = VIRT_FREQ(cur_cluster, rate); + + return rate; +} + +static unsigned int ve_spc_cpufreq_get_rate(unsigned int cpu) +{ + if (is_bL_switching_enabled()) + return per_cpu(cpu_last_req_freq, cpu); + else + return clk_get_cpu_rate(cpu); +} + +static unsigned int +ve_spc_cpufreq_set_rate(u32 cpu, u32 old_cluster, u32 new_cluster, u32 rate) +{ + u32 new_rate, prev_rate; + int ret; + bool bLs = is_bL_switching_enabled(); + + mutex_lock(&cluster_lock[new_cluster]); + + if (bLs) { + prev_rate = per_cpu(cpu_last_req_freq, cpu); + per_cpu(cpu_last_req_freq, cpu) = rate; + per_cpu(physical_cluster, cpu) = new_cluster; + + new_rate = find_cluster_maxfreq(new_cluster); + new_rate = ACTUAL_FREQ(new_cluster, new_rate); + } else { + new_rate = rate; + } + + ret = clk_set_rate(clk[new_cluster], new_rate * 1000); + if (!ret) { + /* + * FIXME: clk_set_rate hasn't returned an error here however it + * may be that clk_change_rate failed due to hardware or + * firmware issues and wasn't able to report that due to the + * current design of the clk core layer. To work around this + * problem we will read back the clock rate and check it is + * correct. This needs to be removed once clk core is fixed. + */ + if (clk_get_rate(clk[new_cluster]) != new_rate * 1000) + ret = -EIO; + } + + if (WARN_ON(ret)) { + if (bLs) { + per_cpu(cpu_last_req_freq, cpu) = prev_rate; + per_cpu(physical_cluster, cpu) = old_cluster; + } + + mutex_unlock(&cluster_lock[new_cluster]); + + return ret; + } + + mutex_unlock(&cluster_lock[new_cluster]); + + /* Recalc freq for old cluster when switching clusters */ + if (old_cluster != new_cluster) { + /* Switch cluster */ + bL_switch_request(cpu, new_cluster); + + mutex_lock(&cluster_lock[old_cluster]); + + /* Set freq of old cluster if there are cpus left on it */ + new_rate = find_cluster_maxfreq(old_cluster); + new_rate = ACTUAL_FREQ(old_cluster, new_rate); + + if (new_rate && + clk_set_rate(clk[old_cluster], new_rate * 1000)) { + pr_err("%s: clk_set_rate failed: %d, old cluster: %d\n", + __func__, ret, old_cluster); + } + mutex_unlock(&cluster_lock[old_cluster]); + } + + return 0; +} + +/* Set clock frequency */ +static int ve_spc_cpufreq_set_target(struct cpufreq_policy *policy, + unsigned int index) +{ + u32 cpu = policy->cpu, cur_cluster, new_cluster, actual_cluster; + unsigned int freqs_new; + + cur_cluster = cpu_to_cluster(cpu); + new_cluster = actual_cluster = per_cpu(physical_cluster, cpu); + + freqs_new = freq_table[cur_cluster][index].frequency; + + if (is_bL_switching_enabled()) { + if (actual_cluster == A15_CLUSTER && freqs_new < clk_big_min) + new_cluster = A7_CLUSTER; + else if (actual_cluster == A7_CLUSTER && + freqs_new > clk_little_max) + new_cluster = A15_CLUSTER; + } + + return ve_spc_cpufreq_set_rate(cpu, actual_cluster, new_cluster, + freqs_new); +} + +static inline u32 get_table_count(struct cpufreq_frequency_table *table) +{ + int count; + + for (count = 0; table[count].frequency != CPUFREQ_TABLE_END; count++) + ; + + return count; +} + +/* get the minimum frequency in the cpufreq_frequency_table */ +static inline u32 get_table_min(struct cpufreq_frequency_table *table) +{ + struct cpufreq_frequency_table *pos; + u32 min_freq = ~0; + + cpufreq_for_each_entry(pos, table) + if (pos->frequency < min_freq) + min_freq = pos->frequency; + return min_freq; +} + +/* get the maximum frequency in the cpufreq_frequency_table */ +static inline u32 get_table_max(struct cpufreq_frequency_table *table) +{ + struct cpufreq_frequency_table *pos; + u32 max_freq = 0; + + cpufreq_for_each_entry(pos, table) + if (pos->frequency > max_freq) + max_freq = pos->frequency; + return max_freq; +} + +static bool search_frequency(struct cpufreq_frequency_table *table, int size, + unsigned int freq) +{ + int count; + + for (count = 0; count < size; count++) { + if (table[count].frequency == freq) + return true; + } + + return false; +} + +static int merge_cluster_tables(void) +{ + int i, j, k = 0, count = 1; + struct cpufreq_frequency_table *table; + + for (i = 0; i < MAX_CLUSTERS; i++) + count += get_table_count(freq_table[i]); + + table = kcalloc(count, sizeof(*table), GFP_KERNEL); + if (!table) + return -ENOMEM; + + freq_table[MAX_CLUSTERS] = table; + + /* Add in reverse order to get freqs in increasing order */ + for (i = MAX_CLUSTERS - 1; i >= 0; i--, count = k) { + for (j = 0; freq_table[i][j].frequency != CPUFREQ_TABLE_END; + j++) { + if (i == A15_CLUSTER && + search_frequency(table, count, freq_table[i][j].frequency)) + continue; /* skip duplicates */ + table[k++].frequency = + VIRT_FREQ(i, freq_table[i][j].frequency); + } + } + + table[k].driver_data = k; + table[k].frequency = CPUFREQ_TABLE_END; + + return 0; +} + +static void _put_cluster_clk_and_freq_table(struct device *cpu_dev, + const struct cpumask *cpumask) +{ + u32 cluster = raw_cpu_to_cluster(cpu_dev->id); + + if (!freq_table[cluster]) + return; + + clk_put(clk[cluster]); + dev_pm_opp_free_cpufreq_table(cpu_dev, &freq_table[cluster]); +} + +static void put_cluster_clk_and_freq_table(struct device *cpu_dev, + const struct cpumask *cpumask) +{ + u32 cluster = cpu_to_cluster(cpu_dev->id); + int i; + + if (atomic_dec_return(&cluster_usage[cluster])) + return; + + if (cluster < MAX_CLUSTERS) + return _put_cluster_clk_and_freq_table(cpu_dev, cpumask); + + for_each_present_cpu(i) { + struct device *cdev = get_cpu_device(i); + + if (!cdev) + return; + + _put_cluster_clk_and_freq_table(cdev, cpumask); + } + + /* free virtual table */ + kfree(freq_table[cluster]); +} + +static int _get_cluster_clk_and_freq_table(struct device *cpu_dev, + const struct cpumask *cpumask) +{ + u32 cluster = raw_cpu_to_cluster(cpu_dev->id); + int ret; + + if (freq_table[cluster]) + return 0; + + /* + * platform specific SPC code must initialise the opp table + * so just check if the OPP count is non-zero + */ + ret = dev_pm_opp_get_opp_count(cpu_dev) <= 0; + if (ret) + goto out; + + ret = dev_pm_opp_init_cpufreq_table(cpu_dev, &freq_table[cluster]); + if (ret) + goto out; + + clk[cluster] = clk_get(cpu_dev, NULL); + if (!IS_ERR(clk[cluster])) + return 0; + + dev_err(cpu_dev, "%s: Failed to get clk for cpu: %d, cluster: %d\n", + __func__, cpu_dev->id, cluster); + ret = PTR_ERR(clk[cluster]); + dev_pm_opp_free_cpufreq_table(cpu_dev, &freq_table[cluster]); + +out: + dev_err(cpu_dev, "%s: Failed to get data for cluster: %d\n", __func__, + cluster); + return ret; +} + +static int get_cluster_clk_and_freq_table(struct device *cpu_dev, + const struct cpumask *cpumask) +{ + u32 cluster = cpu_to_cluster(cpu_dev->id); + int i, ret; + + if (atomic_inc_return(&cluster_usage[cluster]) != 1) + return 0; + + if (cluster < MAX_CLUSTERS) { + ret = _get_cluster_clk_and_freq_table(cpu_dev, cpumask); + if (ret) + atomic_dec(&cluster_usage[cluster]); + return ret; + } + + /* + * Get data for all clusters and fill virtual cluster with a merge of + * both + */ + for_each_present_cpu(i) { + struct device *cdev = get_cpu_device(i); + + if (!cdev) + return -ENODEV; + + ret = _get_cluster_clk_and_freq_table(cdev, cpumask); + if (ret) + goto put_clusters; + } + + ret = merge_cluster_tables(); + if (ret) + goto put_clusters; + + /* Assuming 2 cluster, set clk_big_min and clk_little_max */ + clk_big_min = get_table_min(freq_table[A15_CLUSTER]); + clk_little_max = VIRT_FREQ(A7_CLUSTER, + get_table_max(freq_table[A7_CLUSTER])); + + return 0; + +put_clusters: + for_each_present_cpu(i) { + struct device *cdev = get_cpu_device(i); + + if (!cdev) + return -ENODEV; + + _put_cluster_clk_and_freq_table(cdev, cpumask); + } + + atomic_dec(&cluster_usage[cluster]); + + return ret; +} + +/* Per-CPU initialization */ +static int ve_spc_cpufreq_init(struct cpufreq_policy *policy) +{ + u32 cur_cluster = cpu_to_cluster(policy->cpu); + struct device *cpu_dev; + int ret; + + cpu_dev = get_cpu_device(policy->cpu); + if (!cpu_dev) { + pr_err("%s: failed to get cpu%d device\n", __func__, + policy->cpu); + return -ENODEV; + } + + if (cur_cluster < MAX_CLUSTERS) { + int cpu; + + dev_pm_opp_get_sharing_cpus(cpu_dev, policy->cpus); + + for_each_cpu(cpu, policy->cpus) + per_cpu(physical_cluster, cpu) = cur_cluster; + } else { + /* Assumption: during init, we are always running on A15 */ + per_cpu(physical_cluster, policy->cpu) = A15_CLUSTER; + } + + ret = get_cluster_clk_and_freq_table(cpu_dev, policy->cpus); + if (ret) + return ret; + + policy->freq_table = freq_table[cur_cluster]; + policy->cpuinfo.transition_latency = 1000000; /* 1 ms */ + + if (is_bL_switching_enabled()) + per_cpu(cpu_last_req_freq, policy->cpu) = + clk_get_cpu_rate(policy->cpu); + + dev_info(cpu_dev, "%s: CPU %d initialized\n", __func__, policy->cpu); + return 0; +} + +static void ve_spc_cpufreq_exit(struct cpufreq_policy *policy) +{ + struct device *cpu_dev; + + cpu_dev = get_cpu_device(policy->cpu); + if (!cpu_dev) { + pr_err("%s: failed to get cpu%d device\n", __func__, + policy->cpu); + return; + } + + put_cluster_clk_and_freq_table(cpu_dev, policy->related_cpus); +} + +static struct cpufreq_driver ve_spc_cpufreq_driver = { + .name = "vexpress-spc", + .flags = CPUFREQ_HAVE_GOVERNOR_PER_POLICY | + CPUFREQ_NEED_INITIAL_FREQ_CHECK, + .verify = cpufreq_generic_frequency_table_verify, + .target_index = ve_spc_cpufreq_set_target, + .get = ve_spc_cpufreq_get_rate, + .init = ve_spc_cpufreq_init, + .exit = ve_spc_cpufreq_exit, + .register_em = cpufreq_register_em_with_opp, +}; + +#ifdef CONFIG_BL_SWITCHER +static int bL_cpufreq_switcher_notifier(struct notifier_block *nfb, + unsigned long action, void *_arg) +{ + pr_debug("%s: action: %ld\n", __func__, action); + + switch (action) { + case BL_NOTIFY_PRE_ENABLE: + case BL_NOTIFY_PRE_DISABLE: + cpufreq_unregister_driver(&ve_spc_cpufreq_driver); + break; + + case BL_NOTIFY_POST_ENABLE: + set_switching_enabled(true); + cpufreq_register_driver(&ve_spc_cpufreq_driver); + break; + + case BL_NOTIFY_POST_DISABLE: + set_switching_enabled(false); + cpufreq_register_driver(&ve_spc_cpufreq_driver); + break; + + default: + return NOTIFY_DONE; + } + + return NOTIFY_OK; +} + +static struct notifier_block bL_switcher_notifier = { + .notifier_call = bL_cpufreq_switcher_notifier, +}; + +static int __bLs_register_notifier(void) +{ + return bL_switcher_register_notifier(&bL_switcher_notifier); +} + +static int __bLs_unregister_notifier(void) +{ + return bL_switcher_unregister_notifier(&bL_switcher_notifier); +} +#else +static int __bLs_register_notifier(void) { return 0; } +static int __bLs_unregister_notifier(void) { return 0; } +#endif + +static int ve_spc_cpufreq_probe(struct platform_device *pdev) +{ + int ret, i; + + set_switching_enabled(bL_switcher_get_enabled()); + + for (i = 0; i < MAX_CLUSTERS; i++) + mutex_init(&cluster_lock[i]); + + if (!is_bL_switching_enabled()) + ve_spc_cpufreq_driver.flags |= CPUFREQ_IS_COOLING_DEV; + + ret = cpufreq_register_driver(&ve_spc_cpufreq_driver); + if (ret) { + pr_info("%s: Failed registering platform driver: %s, err: %d\n", + __func__, ve_spc_cpufreq_driver.name, ret); + } else { + ret = __bLs_register_notifier(); + if (ret) + cpufreq_unregister_driver(&ve_spc_cpufreq_driver); + else + pr_info("%s: Registered platform driver: %s\n", + __func__, ve_spc_cpufreq_driver.name); + } + + bL_switcher_put_enabled(); + return ret; +} + +static void ve_spc_cpufreq_remove(struct platform_device *pdev) +{ + bL_switcher_get_enabled(); + __bLs_unregister_notifier(); + cpufreq_unregister_driver(&ve_spc_cpufreq_driver); + bL_switcher_put_enabled(); + pr_info("%s: Un-registered platform driver: %s\n", __func__, + ve_spc_cpufreq_driver.name); +} + +static struct platform_driver ve_spc_cpufreq_platdrv = { + .driver = { + .name = "vexpress-spc-cpufreq", + }, + .probe = ve_spc_cpufreq_probe, + .remove = ve_spc_cpufreq_remove, +}; +module_platform_driver(ve_spc_cpufreq_platdrv); + +MODULE_ALIAS("platform:vexpress-spc-cpufreq"); +MODULE_AUTHOR("Viresh Kumar <viresh.kumar@linaro.org>"); +MODULE_AUTHOR("Sudeep Holla <sudeep.holla@arm.com>"); +MODULE_DESCRIPTION("Vexpress SPC ARM big LITTLE cpufreq driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/cpufreq/virtual-cpufreq.c b/drivers/cpufreq/virtual-cpufreq.c new file mode 100644 index 000000000000..6ffa16d239b2 --- /dev/null +++ b/drivers/cpufreq/virtual-cpufreq.c @@ -0,0 +1,332 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2024 Google LLC + */ + +#include <linux/arch_topology.h> +#include <linux/cpufreq.h> +#include <linux/init.h> +#include <linux/sched.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/of_address.h> +#include <linux/of_platform.h> +#include <linux/platform_device.h> +#include <linux/slab.h> + +/* + * CPU0..CPUn + * +-------------+-------------------------------+--------+-------+ + * | Register | Description | Offset | Len | + * +-------------+-------------------------------+--------+-------+ + * | cur_perf | read this register to get | 0x0 | 0x4 | + * | | the current perf (integer val | | | + * | | representing perf relative to | | | + * | | max performance) | | | + * | | that vCPU is running at | | | + * +-------------+-------------------------------+--------+-------+ + * | set_perf | write to this register to set | 0x4 | 0x4 | + * | | perf value of the vCPU | | | + * +-------------+-------------------------------+--------+-------+ + * | perftbl_len | number of entries in perf | 0x8 | 0x4 | + * | | table. A single entry in the | | | + * | | perf table denotes no table | | | + * | | and the entry contains | | | + * | | the maximum perf value | | | + * | | that this vCPU supports. | | | + * | | The guest can request any | | | + * | | value between 1 and max perf | | | + * | | when perftbls are not used. | | | + * +---------------------------------------------+--------+-------+ + * | perftbl_sel | write to this register to | 0xc | 0x4 | + * | | select perf table entry to | | | + * | | read from | | | + * +---------------------------------------------+--------+-------+ + * | perftbl_rd | read this register to get | 0x10 | 0x4 | + * | | perf value of the selected | | | + * | | entry based on perftbl_sel | | | + * +---------------------------------------------+--------+-------+ + * | perf_domain | performance domain number | 0x14 | 0x4 | + * | | that this vCPU belongs to. | | | + * | | vCPUs sharing the same perf | | | + * | | domain number are part of the | | | + * | | same performance domain. | | | + * +-------------+-------------------------------+--------+-------+ + */ + +#define REG_CUR_PERF_STATE_OFFSET 0x0 +#define REG_SET_PERF_STATE_OFFSET 0x4 +#define REG_PERFTBL_LEN_OFFSET 0x8 +#define REG_PERFTBL_SEL_OFFSET 0xc +#define REG_PERFTBL_RD_OFFSET 0x10 +#define REG_PERF_DOMAIN_OFFSET 0x14 +#define PER_CPU_OFFSET 0x1000 + +#define PERFTBL_MAX_ENTRIES 64U + +static void __iomem *base; +static DEFINE_PER_CPU(u32, perftbl_num_entries); + +static void virt_scale_freq_tick(void) +{ + int cpu = smp_processor_id(); + u32 max_freq = (u32)cpufreq_get_hw_max_freq(cpu); + u64 cur_freq; + unsigned long scale; + + cur_freq = (u64)readl_relaxed(base + cpu * PER_CPU_OFFSET + + REG_CUR_PERF_STATE_OFFSET); + + cur_freq <<= SCHED_CAPACITY_SHIFT; + scale = (unsigned long)div_u64(cur_freq, max_freq); + scale = min(scale, SCHED_CAPACITY_SCALE); + + this_cpu_write(arch_freq_scale, scale); +} + +static struct scale_freq_data virt_sfd = { + .source = SCALE_FREQ_SOURCE_VIRT, + .set_freq_scale = virt_scale_freq_tick, +}; + +static unsigned int virt_cpufreq_set_perf(struct cpufreq_policy *policy, + unsigned int target_freq) +{ + writel_relaxed(target_freq, + base + policy->cpu * PER_CPU_OFFSET + REG_SET_PERF_STATE_OFFSET); + return 0; +} + +static unsigned int virt_cpufreq_fast_switch(struct cpufreq_policy *policy, + unsigned int target_freq) +{ + virt_cpufreq_set_perf(policy, target_freq); + return target_freq; +} + +static u32 virt_cpufreq_get_perftbl_entry(int cpu, u32 idx) +{ + writel_relaxed(idx, base + cpu * PER_CPU_OFFSET + + REG_PERFTBL_SEL_OFFSET); + return readl_relaxed(base + cpu * PER_CPU_OFFSET + + REG_PERFTBL_RD_OFFSET); +} + +static int virt_cpufreq_target(struct cpufreq_policy *policy, + unsigned int target_freq, + unsigned int relation) +{ + struct cpufreq_freqs freqs; + int ret = 0; + + freqs.old = policy->cur; + freqs.new = target_freq; + + cpufreq_freq_transition_begin(policy, &freqs); + ret = virt_cpufreq_set_perf(policy, target_freq); + cpufreq_freq_transition_end(policy, &freqs, ret != 0); + + return ret; +} + +static int virt_cpufreq_get_sharing_cpus(struct cpufreq_policy *policy) +{ + u32 cur_perf_domain, perf_domain; + struct device *cpu_dev; + int cpu; + + cur_perf_domain = readl_relaxed(base + policy->cpu * + PER_CPU_OFFSET + REG_PERF_DOMAIN_OFFSET); + + for_each_present_cpu(cpu) { + cpu_dev = get_cpu_device(cpu); + if (!cpu_dev) + continue; + + perf_domain = readl_relaxed(base + cpu * + PER_CPU_OFFSET + REG_PERF_DOMAIN_OFFSET); + + if (perf_domain == cur_perf_domain) + cpumask_set_cpu(cpu, policy->cpus); + } + + return 0; +} + +static int virt_cpufreq_get_freq_info(struct cpufreq_policy *policy) +{ + struct cpufreq_frequency_table *table; + u32 num_perftbl_entries, idx; + + num_perftbl_entries = per_cpu(perftbl_num_entries, policy->cpu); + + if (num_perftbl_entries == 1) { + policy->cpuinfo.min_freq = 1; + policy->cpuinfo.max_freq = virt_cpufreq_get_perftbl_entry(policy->cpu, 0); + + policy->min = policy->cpuinfo.min_freq; + policy->max = policy->cpuinfo.max_freq; + + policy->cur = policy->max; + return 0; + } + + table = kcalloc(num_perftbl_entries + 1, sizeof(*table), GFP_KERNEL); + if (!table) + return -ENOMEM; + + for (idx = 0; idx < num_perftbl_entries; idx++) + table[idx].frequency = virt_cpufreq_get_perftbl_entry(policy->cpu, idx); + + table[idx].frequency = CPUFREQ_TABLE_END; + policy->freq_table = table; + + return 0; +} + +static int virt_cpufreq_cpu_init(struct cpufreq_policy *policy) +{ + struct device *cpu_dev; + int ret; + + cpu_dev = get_cpu_device(policy->cpu); + if (!cpu_dev) + return -ENODEV; + + ret = virt_cpufreq_get_freq_info(policy); + if (ret) { + dev_warn(cpu_dev, "failed to get cpufreq info\n"); + return ret; + } + + ret = virt_cpufreq_get_sharing_cpus(policy); + if (ret) { + dev_warn(cpu_dev, "failed to get sharing cpumask\n"); + return ret; + } + + /* + * To simplify and improve latency of handling frequency requests on + * the host side, this ensures that the vCPU thread triggering the MMIO + * abort is the same thread whose performance constraints (Ex. uclamp + * settings) need to be updated. This simplifies the VMM (Virtual + * Machine Manager) having to find the correct vCPU thread and/or + * facing permission issues when configuring other threads. + */ + policy->dvfs_possible_from_any_cpu = false; + policy->fast_switch_possible = true; + + /* + * Using the default SCALE_FREQ_SOURCE_CPUFREQ is insufficient since + * the actual physical CPU frequency may not match requested frequency + * from the vCPU thread due to frequency update latencies or other + * inputs to the physical CPU frequency selection. This additional FIE + * source allows for more accurate freq_scale updates and only takes + * effect if another FIE source such as AMUs have not been registered. + */ + topology_set_scale_freq_source(&virt_sfd, policy->cpus); + + return 0; +} + +static void virt_cpufreq_cpu_exit(struct cpufreq_policy *policy) +{ + topology_clear_scale_freq_source(SCALE_FREQ_SOURCE_VIRT, policy->related_cpus); + kfree(policy->freq_table); +} + +static int virt_cpufreq_online(struct cpufreq_policy *policy) +{ + /* Nothing to restore. */ + return 0; +} + +static int virt_cpufreq_offline(struct cpufreq_policy *policy) +{ + /* Dummy offline() to avoid exit() being called and freeing resources. */ + return 0; +} + +static int virt_cpufreq_verify_policy(struct cpufreq_policy_data *policy) +{ + if (policy->freq_table) + return cpufreq_frequency_table_verify(policy); + + cpufreq_verify_within_cpu_limits(policy); + return 0; +} + +static struct cpufreq_driver cpufreq_virt_driver = { + .name = "virt-cpufreq", + .init = virt_cpufreq_cpu_init, + .exit = virt_cpufreq_cpu_exit, + .online = virt_cpufreq_online, + .offline = virt_cpufreq_offline, + .verify = virt_cpufreq_verify_policy, + .target = virt_cpufreq_target, + .fast_switch = virt_cpufreq_fast_switch, +}; + +static int virt_cpufreq_driver_probe(struct platform_device *pdev) +{ + u32 num_perftbl_entries; + int ret, cpu; + + base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(base)) + return PTR_ERR(base); + + for_each_possible_cpu(cpu) { + num_perftbl_entries = readl_relaxed(base + cpu * PER_CPU_OFFSET + + REG_PERFTBL_LEN_OFFSET); + + if (!num_perftbl_entries || num_perftbl_entries > PERFTBL_MAX_ENTRIES) + return -ENODEV; + + per_cpu(perftbl_num_entries, cpu) = num_perftbl_entries; + } + + ret = cpufreq_register_driver(&cpufreq_virt_driver); + if (ret) { + dev_err(&pdev->dev, "Virtual CPUFreq driver failed to register: %d\n", ret); + return ret; + } + + dev_dbg(&pdev->dev, "Virtual CPUFreq driver initialized\n"); + return 0; +} + +static void virt_cpufreq_driver_remove(struct platform_device *pdev) +{ + cpufreq_unregister_driver(&cpufreq_virt_driver); +} + +static const struct of_device_id virt_cpufreq_match[] = { + { .compatible = "qemu,virtual-cpufreq", .data = NULL}, + {} +}; +MODULE_DEVICE_TABLE(of, virt_cpufreq_match); + +static struct platform_driver virt_cpufreq_driver = { + .probe = virt_cpufreq_driver_probe, + .remove = virt_cpufreq_driver_remove, + .driver = { + .name = "virt-cpufreq", + .of_match_table = virt_cpufreq_match, + }, +}; + +static int __init virt_cpufreq_init(void) +{ + return platform_driver_register(&virt_cpufreq_driver); +} +postcore_initcall(virt_cpufreq_init); + +static void __exit virt_cpufreq_exit(void) +{ + platform_driver_unregister(&virt_cpufreq_driver); +} +module_exit(virt_cpufreq_exit); + +MODULE_DESCRIPTION("Virtual cpufreq driver"); +MODULE_LICENSE("GPL"); |