diff options
Diffstat (limited to 'drivers/idle/intel_idle.c')
| -rw-r--r-- | drivers/idle/intel_idle.c | 271 |
1 files changed, 200 insertions, 71 deletions
diff --git a/drivers/idle/intel_idle.c b/drivers/idle/intel_idle.c index bcf1198e8991..0fdb1d1316c4 100644 --- a/drivers/idle/intel_idle.c +++ b/drivers/idle/intel_idle.c @@ -51,10 +51,12 @@ #include <linux/notifier.h> #include <linux/cpu.h> #include <linux/moduleparam.h> +#include <asm/cpuid.h> #include <asm/cpu_device_id.h> #include <asm/intel-family.h> #include <asm/mwait.h> #include <asm/spec-ctrl.h> +#include <asm/tsc.h> #include <asm/fpu/api.h> #define INTEL_IDLE_VERSION "0.5.1" @@ -121,6 +123,12 @@ static unsigned int mwait_substates __initdata; #define CPUIDLE_FLAG_INIT_XSTATE BIT(17) /* + * Ignore the sub-state when matching mwait hints between the ACPI _CST and + * custom tables. + */ +#define CPUIDLE_FLAG_PARTIAL_HINT_MATCH BIT(18) + +/* * MWAIT takes an 8-bit "hint" in EAX "suggesting" * the C-state (top nibble) and sub-state (bottom nibble) * 0x00 means "MWAIT(C1)", 0x10 means "MWAIT(C2)" etc. @@ -1022,6 +1030,88 @@ static struct cpuidle_state spr_cstates[] __initdata = { .enter = NULL } }; +static struct cpuidle_state gnr_cstates[] __initdata = { + { + .name = "C1", + .desc = "MWAIT 0x00", + .flags = MWAIT2flg(0x00), + .exit_latency = 1, + .target_residency = 1, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { + .name = "C1E", + .desc = "MWAIT 0x01", + .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, + .exit_latency = 4, + .target_residency = 4, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { + .name = "C6", + .desc = "MWAIT 0x20", + .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | + CPUIDLE_FLAG_INIT_XSTATE | + CPUIDLE_FLAG_PARTIAL_HINT_MATCH, + .exit_latency = 170, + .target_residency = 650, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { + .name = "C6P", + .desc = "MWAIT 0x21", + .flags = MWAIT2flg(0x21) | CPUIDLE_FLAG_TLB_FLUSHED | + CPUIDLE_FLAG_INIT_XSTATE | + CPUIDLE_FLAG_PARTIAL_HINT_MATCH, + .exit_latency = 210, + .target_residency = 1000, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { + .enter = NULL } +}; + +static struct cpuidle_state gnrd_cstates[] __initdata = { + { + .name = "C1", + .desc = "MWAIT 0x00", + .flags = MWAIT2flg(0x00), + .exit_latency = 1, + .target_residency = 1, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { + .name = "C1E", + .desc = "MWAIT 0x01", + .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, + .exit_latency = 4, + .target_residency = 4, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { + .name = "C6", + .desc = "MWAIT 0x20", + .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | + CPUIDLE_FLAG_INIT_XSTATE | + CPUIDLE_FLAG_PARTIAL_HINT_MATCH, + .exit_latency = 220, + .target_residency = 650, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { + .name = "C6P", + .desc = "MWAIT 0x21", + .flags = MWAIT2flg(0x21) | CPUIDLE_FLAG_TLB_FLUSHED | + CPUIDLE_FLAG_INIT_XSTATE | + CPUIDLE_FLAG_PARTIAL_HINT_MATCH, + .exit_latency = 240, + .target_residency = 750, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { + .enter = NULL } +}; + static struct cpuidle_state atom_cstates[] __initdata = { { .name = "C1E", @@ -1315,7 +1405,8 @@ static struct cpuidle_state srf_cstates[] __initdata = { { .name = "C6S", .desc = "MWAIT 0x22", - .flags = MWAIT2flg(0x22) | CPUIDLE_FLAG_TLB_FLUSHED, + .flags = MWAIT2flg(0x22) | CPUIDLE_FLAG_TLB_FLUSHED | + CPUIDLE_FLAG_PARTIAL_HINT_MATCH, .exit_latency = 270, .target_residency = 700, .enter = &intel_idle, @@ -1323,7 +1414,8 @@ static struct cpuidle_state srf_cstates[] __initdata = { { .name = "C6SP", .desc = "MWAIT 0x23", - .flags = MWAIT2flg(0x23) | CPUIDLE_FLAG_TLB_FLUSHED, + .flags = MWAIT2flg(0x23) | CPUIDLE_FLAG_TLB_FLUSHED | + CPUIDLE_FLAG_PARTIAL_HINT_MATCH, .exit_latency = 310, .target_residency = 900, .enter = &intel_idle, @@ -1453,6 +1545,18 @@ static const struct idle_cpu idle_cpu_spr __initconst = { .use_acpi = true, }; +static const struct idle_cpu idle_cpu_gnr __initconst = { + .state_table = gnr_cstates, + .disable_promotion_to_c1e = true, + .use_acpi = true, +}; + +static const struct idle_cpu idle_cpu_gnrd __initconst = { + .state_table = gnrd_cstates, + .disable_promotion_to_c1e = true, + .use_acpi = true, +}; + static const struct idle_cpu idle_cpu_avn __initconst = { .state_table = avn_cstates, .disable_promotion_to_c1e = true, @@ -1475,6 +1579,10 @@ static const struct idle_cpu idle_cpu_dnv __initconst = { .use_acpi = true, }; +static const struct idle_cpu idle_cpu_tmt __initconst = { + .disable_promotion_to_c1e = true, +}; + static const struct idle_cpu idle_cpu_snr __initconst = { .state_table = snr_cstates, .disable_promotion_to_c1e = true, @@ -1494,53 +1602,58 @@ static const struct idle_cpu idle_cpu_srf __initconst = { }; static const struct x86_cpu_id intel_idle_ids[] __initconst = { - X86_MATCH_INTEL_FAM6_MODEL(NEHALEM_EP, &idle_cpu_nhx), - X86_MATCH_INTEL_FAM6_MODEL(NEHALEM, &idle_cpu_nehalem), - X86_MATCH_INTEL_FAM6_MODEL(NEHALEM_G, &idle_cpu_nehalem), - X86_MATCH_INTEL_FAM6_MODEL(WESTMERE, &idle_cpu_nehalem), - X86_MATCH_INTEL_FAM6_MODEL(WESTMERE_EP, &idle_cpu_nhx), - X86_MATCH_INTEL_FAM6_MODEL(NEHALEM_EX, &idle_cpu_nhx), - X86_MATCH_INTEL_FAM6_MODEL(ATOM_BONNELL, &idle_cpu_atom), - X86_MATCH_INTEL_FAM6_MODEL(ATOM_BONNELL_MID, &idle_cpu_lincroft), - X86_MATCH_INTEL_FAM6_MODEL(WESTMERE_EX, &idle_cpu_nhx), - X86_MATCH_INTEL_FAM6_MODEL(SANDYBRIDGE, &idle_cpu_snb), - X86_MATCH_INTEL_FAM6_MODEL(SANDYBRIDGE_X, &idle_cpu_snx), - X86_MATCH_INTEL_FAM6_MODEL(ATOM_SALTWELL, &idle_cpu_atom), - X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT, &idle_cpu_byt), - X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT_MID, &idle_cpu_tangier), - X86_MATCH_INTEL_FAM6_MODEL(ATOM_AIRMONT, &idle_cpu_cht), - X86_MATCH_INTEL_FAM6_MODEL(IVYBRIDGE, &idle_cpu_ivb), - X86_MATCH_INTEL_FAM6_MODEL(IVYBRIDGE_X, &idle_cpu_ivt), - X86_MATCH_INTEL_FAM6_MODEL(HASWELL, &idle_cpu_hsw), - X86_MATCH_INTEL_FAM6_MODEL(HASWELL_X, &idle_cpu_hsx), - X86_MATCH_INTEL_FAM6_MODEL(HASWELL_L, &idle_cpu_hsw), - X86_MATCH_INTEL_FAM6_MODEL(HASWELL_G, &idle_cpu_hsw), - X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT_D, &idle_cpu_avn), - X86_MATCH_INTEL_FAM6_MODEL(BROADWELL, &idle_cpu_bdw), - X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_G, &idle_cpu_bdw), - X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_X, &idle_cpu_bdx), - X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_D, &idle_cpu_bdx), - X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_L, &idle_cpu_skl), - X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE, &idle_cpu_skl), - X86_MATCH_INTEL_FAM6_MODEL(KABYLAKE_L, &idle_cpu_skl), - X86_MATCH_INTEL_FAM6_MODEL(KABYLAKE, &idle_cpu_skl), - X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_X, &idle_cpu_skx), - X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_X, &idle_cpu_icx), - X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_D, &idle_cpu_icx), - X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE, &idle_cpu_adl), - X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE_L, &idle_cpu_adl_l), - X86_MATCH_INTEL_FAM6_MODEL(METEORLAKE_L, &idle_cpu_mtl_l), - X86_MATCH_INTEL_FAM6_MODEL(ATOM_GRACEMONT, &idle_cpu_gmt), - X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X, &idle_cpu_spr), - X86_MATCH_INTEL_FAM6_MODEL(EMERALDRAPIDS_X, &idle_cpu_spr), - X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNL, &idle_cpu_knl), - X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNM, &idle_cpu_knl), - X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT, &idle_cpu_bxt), - X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT_PLUS, &idle_cpu_bxt), - X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT_D, &idle_cpu_dnv), - X86_MATCH_INTEL_FAM6_MODEL(ATOM_TREMONT_D, &idle_cpu_snr), - X86_MATCH_INTEL_FAM6_MODEL(ATOM_CRESTMONT, &idle_cpu_grr), - X86_MATCH_INTEL_FAM6_MODEL(ATOM_CRESTMONT_X, &idle_cpu_srf), + X86_MATCH_VFM(INTEL_NEHALEM_EP, &idle_cpu_nhx), + X86_MATCH_VFM(INTEL_NEHALEM, &idle_cpu_nehalem), + X86_MATCH_VFM(INTEL_NEHALEM_G, &idle_cpu_nehalem), + X86_MATCH_VFM(INTEL_WESTMERE, &idle_cpu_nehalem), + X86_MATCH_VFM(INTEL_WESTMERE_EP, &idle_cpu_nhx), + X86_MATCH_VFM(INTEL_NEHALEM_EX, &idle_cpu_nhx), + X86_MATCH_VFM(INTEL_ATOM_BONNELL, &idle_cpu_atom), + X86_MATCH_VFM(INTEL_ATOM_BONNELL_MID, &idle_cpu_lincroft), + X86_MATCH_VFM(INTEL_WESTMERE_EX, &idle_cpu_nhx), + X86_MATCH_VFM(INTEL_SANDYBRIDGE, &idle_cpu_snb), + X86_MATCH_VFM(INTEL_SANDYBRIDGE_X, &idle_cpu_snx), + X86_MATCH_VFM(INTEL_ATOM_SALTWELL, &idle_cpu_atom), + X86_MATCH_VFM(INTEL_ATOM_SILVERMONT, &idle_cpu_byt), + X86_MATCH_VFM(INTEL_ATOM_SILVERMONT_MID, &idle_cpu_tangier), + X86_MATCH_VFM(INTEL_ATOM_AIRMONT, &idle_cpu_cht), + X86_MATCH_VFM(INTEL_IVYBRIDGE, &idle_cpu_ivb), + X86_MATCH_VFM(INTEL_IVYBRIDGE_X, &idle_cpu_ivt), + X86_MATCH_VFM(INTEL_HASWELL, &idle_cpu_hsw), + X86_MATCH_VFM(INTEL_HASWELL_X, &idle_cpu_hsx), + X86_MATCH_VFM(INTEL_HASWELL_L, &idle_cpu_hsw), + X86_MATCH_VFM(INTEL_HASWELL_G, &idle_cpu_hsw), + X86_MATCH_VFM(INTEL_ATOM_SILVERMONT_D, &idle_cpu_avn), + X86_MATCH_VFM(INTEL_BROADWELL, &idle_cpu_bdw), + X86_MATCH_VFM(INTEL_BROADWELL_G, &idle_cpu_bdw), + X86_MATCH_VFM(INTEL_BROADWELL_X, &idle_cpu_bdx), + X86_MATCH_VFM(INTEL_BROADWELL_D, &idle_cpu_bdx), + X86_MATCH_VFM(INTEL_SKYLAKE_L, &idle_cpu_skl), + X86_MATCH_VFM(INTEL_SKYLAKE, &idle_cpu_skl), + X86_MATCH_VFM(INTEL_KABYLAKE_L, &idle_cpu_skl), + X86_MATCH_VFM(INTEL_KABYLAKE, &idle_cpu_skl), + X86_MATCH_VFM(INTEL_SKYLAKE_X, &idle_cpu_skx), + X86_MATCH_VFM(INTEL_ICELAKE_X, &idle_cpu_icx), + X86_MATCH_VFM(INTEL_ICELAKE_D, &idle_cpu_icx), + X86_MATCH_VFM(INTEL_ALDERLAKE, &idle_cpu_adl), + X86_MATCH_VFM(INTEL_ALDERLAKE_L, &idle_cpu_adl_l), + X86_MATCH_VFM(INTEL_METEORLAKE_L, &idle_cpu_mtl_l), + X86_MATCH_VFM(INTEL_ATOM_GRACEMONT, &idle_cpu_gmt), + X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X, &idle_cpu_spr), + X86_MATCH_VFM(INTEL_EMERALDRAPIDS_X, &idle_cpu_spr), + X86_MATCH_VFM(INTEL_GRANITERAPIDS_X, &idle_cpu_gnr), + X86_MATCH_VFM(INTEL_GRANITERAPIDS_D, &idle_cpu_gnrd), + X86_MATCH_VFM(INTEL_XEON_PHI_KNL, &idle_cpu_knl), + X86_MATCH_VFM(INTEL_XEON_PHI_KNM, &idle_cpu_knl), + X86_MATCH_VFM(INTEL_ATOM_GOLDMONT, &idle_cpu_bxt), + X86_MATCH_VFM(INTEL_ATOM_GOLDMONT_PLUS, &idle_cpu_bxt), + X86_MATCH_VFM(INTEL_ATOM_GOLDMONT_D, &idle_cpu_dnv), + X86_MATCH_VFM(INTEL_ATOM_TREMONT, &idle_cpu_tmt), + X86_MATCH_VFM(INTEL_ATOM_TREMONT_L, &idle_cpu_tmt), + X86_MATCH_VFM(INTEL_ATOM_TREMONT_D, &idle_cpu_snr), + X86_MATCH_VFM(INTEL_ATOM_CRESTMONT, &idle_cpu_grr), + X86_MATCH_VFM(INTEL_ATOM_CRESTMONT_X, &idle_cpu_srf), + X86_MATCH_VFM(INTEL_ATOM_DARKMONT_X, &idle_cpu_srf), {} }; @@ -1687,12 +1800,15 @@ static void __init intel_idle_init_cstates_acpi(struct cpuidle_driver *drv) if (intel_idle_state_needs_timer_stop(state)) state->flags |= CPUIDLE_FLAG_TIMER_STOP; + if (cx->type > ACPI_STATE_C1 && !boot_cpu_has(X86_FEATURE_NONSTOP_TSC)) + mark_tsc_unstable("TSC halts in idle"); + state->enter = intel_idle; state->enter_s2idle = intel_idle_s2idle; } } -static bool __init intel_idle_off_by_default(u32 mwait_hint) +static bool __init intel_idle_off_by_default(unsigned int flags, u32 mwait_hint) { int cstate, limit; @@ -1709,7 +1825,15 @@ static bool __init intel_idle_off_by_default(u32 mwait_hint) * the interesting states are ACPI_CSTATE_FFH. */ for (cstate = 1; cstate < limit; cstate++) { - if (acpi_state_table.states[cstate].address == mwait_hint) + u32 acpi_hint = acpi_state_table.states[cstate].address; + u32 table_hint = mwait_hint; + + if (flags & CPUIDLE_FLAG_PARTIAL_HINT_MATCH) { + acpi_hint &= ~MWAIT_SUBSTATE_MASK; + table_hint &= ~MWAIT_SUBSTATE_MASK; + } + + if (acpi_hint == table_hint) return false; } return true; @@ -1719,7 +1843,10 @@ static bool __init intel_idle_off_by_default(u32 mwait_hint) static inline bool intel_idle_acpi_cst_extract(void) { return false; } static inline void intel_idle_init_cstates_acpi(struct cpuidle_driver *drv) { } -static inline bool intel_idle_off_by_default(u32 mwait_hint) { return false; } +static inline bool intel_idle_off_by_default(unsigned int flags, u32 mwait_hint) +{ + return false; +} #endif /* !CONFIG_ACPI_PROCESSOR_CSTATE */ /** @@ -1934,7 +2061,8 @@ static void __init spr_idle_state_table_update(void) static bool __init intel_idle_verify_cstate(unsigned int mwait_hint) { - unsigned int mwait_cstate = MWAIT_HINT2CSTATE(mwait_hint) + 1; + unsigned int mwait_cstate = (MWAIT_HINT2CSTATE(mwait_hint) + 1) & + MWAIT_CSTATE_MASK; unsigned int num_substates = (mwait_substates >> mwait_cstate * 4) & MWAIT_SUBSTATE_MASK; @@ -1989,27 +2117,27 @@ static void __init intel_idle_init_cstates_icpu(struct cpuidle_driver *drv) { int cstate; - switch (boot_cpu_data.x86_model) { - case INTEL_FAM6_IVYBRIDGE_X: + switch (boot_cpu_data.x86_vfm) { + case INTEL_IVYBRIDGE_X: ivt_idle_state_table_update(); break; - case INTEL_FAM6_ATOM_GOLDMONT: - case INTEL_FAM6_ATOM_GOLDMONT_PLUS: + case INTEL_ATOM_GOLDMONT: + case INTEL_ATOM_GOLDMONT_PLUS: bxt_idle_state_table_update(); break; - case INTEL_FAM6_SKYLAKE: + case INTEL_SKYLAKE: sklh_idle_state_table_update(); break; - case INTEL_FAM6_SKYLAKE_X: + case INTEL_SKYLAKE_X: skx_idle_state_table_update(); break; - case INTEL_FAM6_SAPPHIRERAPIDS_X: - case INTEL_FAM6_EMERALDRAPIDS_X: + case INTEL_SAPPHIRERAPIDS_X: + case INTEL_EMERALDRAPIDS_X: spr_idle_state_table_update(); break; - case INTEL_FAM6_ALDERLAKE: - case INTEL_FAM6_ALDERLAKE_L: - case INTEL_FAM6_ATOM_GRACEMONT: + case INTEL_ALDERLAKE: + case INTEL_ALDERLAKE_L: + case INTEL_ATOM_GRACEMONT: adl_idle_state_table_update(); break; } @@ -2045,7 +2173,7 @@ static void __init intel_idle_init_cstates_icpu(struct cpuidle_driver *drv) if ((disabled_states_mask & BIT(drv->state_count)) || ((icpu->use_acpi || force_use_acpi) && - intel_idle_off_by_default(mwait_hint) && + intel_idle_off_by_default(state->flags, mwait_hint) && !(state->flags & CPUIDLE_FLAG_ALWAYS_ENABLE))) state->flags |= CPUIDLE_FLAG_OFF; @@ -2074,7 +2202,7 @@ static void __init intel_idle_cpuidle_driver_init(struct cpuidle_driver *drv) drv->state_count = 1; - if (icpu) + if (icpu && icpu->state_table) intel_idle_init_cstates_icpu(drv); else intel_idle_init_cstates_acpi(drv); @@ -2194,10 +2322,7 @@ static int __init intel_idle_init(void) return -ENODEV; } - if (boot_cpu_data.cpuid_level < CPUID_MWAIT_LEAF) - return -ENODEV; - - cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &mwait_substates); + cpuid(CPUID_LEAF_MWAIT, &eax, &ebx, &ecx, &mwait_substates); if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) || !(ecx & CPUID5_ECX_INTERRUPT_BREAK) || @@ -2208,7 +2333,11 @@ static int __init intel_idle_init(void) icpu = (const struct idle_cpu *)id->driver_data; if (icpu) { - cpuidle_state_table = icpu->state_table; + if (icpu->state_table) + cpuidle_state_table = icpu->state_table; + else if (!intel_idle_acpi_cst_extract()) + return -ENODEV; + auto_demotion_disable_flags = icpu->auto_demotion_disable_flags; if (icpu->disable_promotion_to_c1e) c1e_promotion = C1E_PROMOTION_DISABLE; |