diff options
| -rw-r--r-- | arch/arm64/include/asm/fpsimd.h | 26 | ||||
| -rw-r--r-- | arch/arm64/kernel/cpufeature.c | 24 | ||||
| -rw-r--r-- | arch/arm64/kernel/fpsimd.c | 7 |
3 files changed, 50 insertions, 7 deletions
diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h index 8df46f186c64..bd7bea92dae0 100644 --- a/arch/arm64/include/asm/fpsimd.h +++ b/arch/arm64/include/asm/fpsimd.h @@ -32,6 +32,32 @@ #define VFP_STATE_SIZE ((32 * 8) + 4) #endif +static inline unsigned long cpacr_save_enable_kernel_sve(void) +{ + unsigned long old = read_sysreg(cpacr_el1); + unsigned long set = CPACR_EL1_FPEN_EL1EN | CPACR_EL1_ZEN_EL1EN; + + write_sysreg(old | set, cpacr_el1); + isb(); + return old; +} + +static inline unsigned long cpacr_save_enable_kernel_sme(void) +{ + unsigned long old = read_sysreg(cpacr_el1); + unsigned long set = CPACR_EL1_FPEN_EL1EN | CPACR_EL1_SMEN_EL1EN; + + write_sysreg(old | set, cpacr_el1); + isb(); + return old; +} + +static inline void cpacr_restore(unsigned long cpacr) +{ + write_sysreg(cpacr, cpacr_el1); + isb(); +} + /* * When we defined the maximum SVE vector length we defined the ABI so * that the maximum vector length included all the reserved for future diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 5add7d06469d..e3e0d3d3bd6b 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -1040,13 +1040,19 @@ void __init init_cpu_features(struct cpuinfo_arm64 *info) if (IS_ENABLED(CONFIG_ARM64_SVE) && id_aa64pfr0_sve(read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1))) { + unsigned long cpacr = cpacr_save_enable_kernel_sve(); + info->reg_zcr = read_zcr_features(); init_cpu_ftr_reg(SYS_ZCR_EL1, info->reg_zcr); vec_init_vq_map(ARM64_VEC_SVE); + + cpacr_restore(cpacr); } if (IS_ENABLED(CONFIG_ARM64_SME) && id_aa64pfr1_sme(read_sanitised_ftr_reg(SYS_ID_AA64PFR1_EL1))) { + unsigned long cpacr = cpacr_save_enable_kernel_sme(); + info->reg_smcr = read_smcr_features(); /* * We mask out SMPS since even if the hardware @@ -1056,6 +1062,8 @@ void __init init_cpu_features(struct cpuinfo_arm64 *info) info->reg_smidr = read_cpuid(SMIDR_EL1) & ~SMIDR_EL1_SMPS; init_cpu_ftr_reg(SYS_SMCR_EL1, info->reg_smcr); vec_init_vq_map(ARM64_VEC_SME); + + cpacr_restore(cpacr); } if (id_aa64pfr1_mte(info->reg_id_aa64pfr1)) @@ -1291,6 +1299,8 @@ void update_cpu_features(int cpu, if (IS_ENABLED(CONFIG_ARM64_SVE) && id_aa64pfr0_sve(read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1))) { + unsigned long cpacr = cpacr_save_enable_kernel_sve(); + info->reg_zcr = read_zcr_features(); taint |= check_update_ftr_reg(SYS_ZCR_EL1, cpu, info->reg_zcr, boot->reg_zcr); @@ -1298,10 +1308,14 @@ void update_cpu_features(int cpu, /* Probe vector lengths */ if (!system_capabilities_finalized()) vec_update_vq_map(ARM64_VEC_SVE); + + cpacr_restore(cpacr); } if (IS_ENABLED(CONFIG_ARM64_SME) && id_aa64pfr1_sme(read_sanitised_ftr_reg(SYS_ID_AA64PFR1_EL1))) { + unsigned long cpacr = cpacr_save_enable_kernel_sme(); + info->reg_smcr = read_smcr_features(); /* * We mask out SMPS since even if the hardware @@ -1315,6 +1329,8 @@ void update_cpu_features(int cpu, /* Probe vector lengths */ if (!system_capabilities_finalized()) vec_update_vq_map(ARM64_VEC_SME); + + cpacr_restore(cpacr); } /* @@ -3174,6 +3190,8 @@ static void verify_local_elf_hwcaps(void) static void verify_sve_features(void) { + unsigned long cpacr = cpacr_save_enable_kernel_sve(); + u64 safe_zcr = read_sanitised_ftr_reg(SYS_ZCR_EL1); u64 zcr = read_zcr_features(); @@ -3186,11 +3204,13 @@ static void verify_sve_features(void) cpu_die_early(); } - /* Add checks on other ZCR bits here if necessary */ + cpacr_restore(cpacr); } static void verify_sme_features(void) { + unsigned long cpacr = cpacr_save_enable_kernel_sme(); + u64 safe_smcr = read_sanitised_ftr_reg(SYS_SMCR_EL1); u64 smcr = read_smcr_features(); @@ -3203,7 +3223,7 @@ static void verify_sme_features(void) cpu_die_early(); } - /* Add checks on other SMCR bits here if necessary */ + cpacr_restore(cpacr); } static void verify_hyp_capabilities(void) diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c index 91e44ac7150f..601b973f90ad 100644 --- a/arch/arm64/kernel/fpsimd.c +++ b/arch/arm64/kernel/fpsimd.c @@ -1174,7 +1174,7 @@ void sve_kernel_enable(const struct arm64_cpu_capabilities *__always_unused p) * Read the pseudo-ZCR used by cpufeatures to identify the supported SVE * vector length. * - * Use only if SVE is present. + * Use only if SVE is present and enabled. * This function clobbers the SVE vector length. */ u64 read_zcr_features(void) @@ -1183,7 +1183,6 @@ u64 read_zcr_features(void) * Set the maximum possible VL, and write zeroes to all other * bits to see if they stick. */ - sve_kernel_enable(NULL); write_sysreg_s(ZCR_ELx_LEN_MASK, SYS_ZCR_EL1); /* Return LEN value that would be written to get the maximum VL */ @@ -1337,13 +1336,11 @@ void fa64_kernel_enable(const struct arm64_cpu_capabilities *__always_unused p) * Read the pseudo-SMCR used by cpufeatures to identify the supported * vector length. * - * Use only if SME is present. + * Use only if SME is present and enabled. * This function clobbers the SME vector length. */ u64 read_smcr_features(void) { - sme_kernel_enable(NULL); - /* * Set the maximum possible VL. */ |