summaryrefslogtreecommitdiff
path: root/lib/crypto/x86/sha1.h
diff options
context:
space:
mode:
Diffstat (limited to 'lib/crypto/x86/sha1.h')
-rw-r--r--lib/crypto/x86/sha1.h74
1 files changed, 74 insertions, 0 deletions
diff --git a/lib/crypto/x86/sha1.h b/lib/crypto/x86/sha1.h
new file mode 100644
index 000000000000..e308379d89bc
--- /dev/null
+++ b/lib/crypto/x86/sha1.h
@@ -0,0 +1,74 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * SHA-1 optimized for x86_64
+ *
+ * Copyright 2025 Google LLC
+ */
+#include <asm/fpu/api.h>
+#include <linux/static_call.h>
+
+DEFINE_STATIC_CALL(sha1_blocks_x86, sha1_blocks_generic);
+
+#define DEFINE_X86_SHA1_FN(c_fn, asm_fn) \
+ asmlinkage void asm_fn(struct sha1_block_state *state, \
+ const u8 *data, size_t nblocks); \
+ static void c_fn(struct sha1_block_state *state, \
+ const u8 *data, size_t nblocks) \
+ { \
+ if (likely(irq_fpu_usable())) { \
+ kernel_fpu_begin(); \
+ asm_fn(state, data, nblocks); \
+ kernel_fpu_end(); \
+ } else { \
+ sha1_blocks_generic(state, data, nblocks); \
+ } \
+ }
+
+DEFINE_X86_SHA1_FN(sha1_blocks_ssse3, sha1_transform_ssse3);
+DEFINE_X86_SHA1_FN(sha1_blocks_avx, sha1_transform_avx);
+DEFINE_X86_SHA1_FN(sha1_blocks_ni, sha1_ni_transform);
+
+#define SHA1_AVX2_BLOCK_OPTSIZE 4 /* optimal 4*64 bytes of SHA1 blocks */
+
+asmlinkage void sha1_transform_avx2(struct sha1_block_state *state,
+ const u8 *data, size_t nblocks);
+static void sha1_blocks_avx2(struct sha1_block_state *state,
+ const u8 *data, size_t nblocks)
+{
+ if (likely(irq_fpu_usable())) {
+ kernel_fpu_begin();
+ /* Select the optimal transform based on the number of blocks */
+ if (nblocks >= SHA1_AVX2_BLOCK_OPTSIZE)
+ sha1_transform_avx2(state, data, nblocks);
+ else
+ sha1_transform_avx(state, data, nblocks);
+ kernel_fpu_end();
+ } else {
+ sha1_blocks_generic(state, data, nblocks);
+ }
+}
+
+static void sha1_blocks(struct sha1_block_state *state,
+ const u8 *data, size_t nblocks)
+{
+ static_call(sha1_blocks_x86)(state, data, nblocks);
+}
+
+#define sha1_mod_init_arch sha1_mod_init_arch
+static inline void sha1_mod_init_arch(void)
+{
+ if (boot_cpu_has(X86_FEATURE_SHA_NI)) {
+ static_call_update(sha1_blocks_x86, sha1_blocks_ni);
+ } else if (cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
+ NULL) &&
+ boot_cpu_has(X86_FEATURE_AVX)) {
+ if (boot_cpu_has(X86_FEATURE_AVX2) &&
+ boot_cpu_has(X86_FEATURE_BMI1) &&
+ boot_cpu_has(X86_FEATURE_BMI2))
+ static_call_update(sha1_blocks_x86, sha1_blocks_avx2);
+ else
+ static_call_update(sha1_blocks_x86, sha1_blocks_avx);
+ } else if (boot_cpu_has(X86_FEATURE_SSSE3)) {
+ static_call_update(sha1_blocks_x86, sha1_blocks_ssse3);
+ }
+}