diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/crc/x86/crc-pclmul-template.h | 31 | ||||
| -rw-r--r-- | lib/crc/x86/crc-t10dif.h | 9 | ||||
| -rw-r--r-- | lib/crc/x86/crc32.h | 10 | ||||
| -rw-r--r-- | lib/crc/x86/crc64.h | 15 |
4 files changed, 44 insertions, 21 deletions
diff --git a/lib/crc/x86/crc-pclmul-template.h b/lib/crc/x86/crc-pclmul-template.h index 51cba520a7db..35c950d7010c 100644 --- a/lib/crc/x86/crc-pclmul-template.h +++ b/lib/crc/x86/crc-pclmul-template.h @@ -25,23 +25,20 @@ crc_t prefix##_vpclmul_avx512(crc_t crc, const u8 *p, size_t len, \ const void *consts_ptr); \ DEFINE_STATIC_CALL(prefix##_pclmul, prefix##_pclmul_sse) -#define INIT_CRC_PCLMUL(prefix) \ -do { \ - if (boot_cpu_has(X86_FEATURE_VPCLMULQDQ) && \ - boot_cpu_has(X86_FEATURE_AVX2) && \ - cpu_has_xfeatures(XFEATURE_MASK_YMM, NULL)) { \ - if (boot_cpu_has(X86_FEATURE_AVX512BW) && \ - boot_cpu_has(X86_FEATURE_AVX512VL) && \ - !boot_cpu_has(X86_FEATURE_PREFER_YMM) && \ - cpu_has_xfeatures(XFEATURE_MASK_AVX512, NULL)) { \ - static_call_update(prefix##_pclmul, \ - prefix##_vpclmul_avx512); \ - } else { \ - static_call_update(prefix##_pclmul, \ - prefix##_vpclmul_avx2); \ - } \ - } \ -} while (0) +static inline bool have_vpclmul(void) +{ + return boot_cpu_has(X86_FEATURE_VPCLMULQDQ) && + boot_cpu_has(X86_FEATURE_AVX2) && + cpu_has_xfeatures(XFEATURE_MASK_YMM, NULL); +} + +static inline bool have_avx512(void) +{ + return boot_cpu_has(X86_FEATURE_AVX512BW) && + boot_cpu_has(X86_FEATURE_AVX512VL) && + !boot_cpu_has(X86_FEATURE_PREFER_YMM) && + cpu_has_xfeatures(XFEATURE_MASK_AVX512, NULL); +} /* * Call a [V]PCLMULQDQ optimized CRC function if the data length is at least 16 diff --git a/lib/crc/x86/crc-t10dif.h b/lib/crc/x86/crc-t10dif.h index eb1f23db4daa..2a02a3026f3f 100644 --- a/lib/crc/x86/crc-t10dif.h +++ b/lib/crc/x86/crc-t10dif.h @@ -23,6 +23,13 @@ static inline void crc_t10dif_mod_init_arch(void) { if (boot_cpu_has(X86_FEATURE_PCLMULQDQ)) { static_branch_enable(&have_pclmulqdq); - INIT_CRC_PCLMUL(crc16_msb); + if (have_vpclmul()) { + if (have_avx512()) + static_call_update(crc16_msb_pclmul, + crc16_msb_vpclmul_avx512); + else + static_call_update(crc16_msb_pclmul, + crc16_msb_vpclmul_avx2); + } } } diff --git a/lib/crc/x86/crc32.h b/lib/crc/x86/crc32.h index 28451d5769c3..ba4dacf23340 100644 --- a/lib/crc/x86/crc32.h +++ b/lib/crc/x86/crc32.h @@ -77,7 +77,15 @@ static inline void crc32_mod_init_arch(void) static_branch_enable(&have_crc32); if (boot_cpu_has(X86_FEATURE_PCLMULQDQ)) { static_branch_enable(&have_pclmulqdq); - INIT_CRC_PCLMUL(crc32_lsb); + if (have_vpclmul()) { + if (have_avx512()) { + static_call_update(crc32_lsb_pclmul, + crc32_lsb_vpclmul_avx512); + } else { + static_call_update(crc32_lsb_pclmul, + crc32_lsb_vpclmul_avx2); + } + } } } diff --git a/lib/crc/x86/crc64.h b/lib/crc/x86/crc64.h index 54aca3a9475c..fde1222c4c58 100644 --- a/lib/crc/x86/crc64.h +++ b/lib/crc/x86/crc64.h @@ -31,7 +31,18 @@ static inline void crc64_mod_init_arch(void) { if (boot_cpu_has(X86_FEATURE_PCLMULQDQ)) { static_branch_enable(&have_pclmulqdq); - INIT_CRC_PCLMUL(crc64_msb); - INIT_CRC_PCLMUL(crc64_lsb); + if (have_vpclmul()) { + if (have_avx512()) { + static_call_update(crc64_msb_pclmul, + crc64_msb_vpclmul_avx512); + static_call_update(crc64_lsb_pclmul, + crc64_lsb_vpclmul_avx512); + } else { + static_call_update(crc64_msb_pclmul, + crc64_msb_vpclmul_avx2); + static_call_update(crc64_lsb_pclmul, + crc64_lsb_vpclmul_avx2); + } + } } } |