diff options
Diffstat (limited to 'arch/s390')
26 files changed, 38 insertions, 1866 deletions
diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 0c16dc443e2f..25a773e6596e 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -75,7 +75,6 @@ config S390 select ARCH_ENABLE_MEMORY_HOTREMOVE select ARCH_ENABLE_SPLIT_PMD_PTLOCK if PGTABLE_LEVELS > 2 select ARCH_HAS_CPU_FINALIZE_INIT - select ARCH_HAS_CRC32 select ARCH_HAS_CURRENT_STACK_POINTER select ARCH_HAS_DEBUG_VIRTUAL select ARCH_HAS_DEBUG_VM_PGTABLE @@ -176,10 +175,10 @@ config S390 select HAVE_ARCH_KCSAN select HAVE_ARCH_KMSAN select HAVE_ARCH_KFENCE + select HAVE_ARCH_KSTACK_ERASE select HAVE_ARCH_RANDOMIZE_KSTACK_OFFSET select HAVE_ARCH_SECCOMP_FILTER select HAVE_ARCH_SOFT_DIRTY - select HAVE_ARCH_STACKLEAK select HAVE_ARCH_TRACEHOOK select HAVE_ARCH_TRANSPARENT_HUGEPAGE select HAVE_ARCH_VMAP_STACK diff --git a/arch/s390/configs/debug_defconfig b/arch/s390/configs/debug_defconfig index 8ecad727497e..a7db7ed28720 100644 --- a/arch/s390/configs/debug_defconfig +++ b/arch/s390/configs/debug_defconfig @@ -804,8 +804,6 @@ CONFIG_CRYPTO_USER_API_HASH=m CONFIG_CRYPTO_USER_API_SKCIPHER=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_USER_API_AEAD=m -CONFIG_CRYPTO_SHA512_S390=m -CONFIG_CRYPTO_SHA1_S390=m CONFIG_CRYPTO_SHA3_256_S390=m CONFIG_CRYPTO_SHA3_512_S390=m CONFIG_CRYPTO_GHASH_S390=m diff --git a/arch/s390/configs/defconfig b/arch/s390/configs/defconfig index c13a77765162..0217ed5616bf 100644 --- a/arch/s390/configs/defconfig +++ b/arch/s390/configs/defconfig @@ -791,8 +791,6 @@ CONFIG_CRYPTO_USER_API_HASH=m CONFIG_CRYPTO_USER_API_SKCIPHER=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_USER_API_AEAD=m -CONFIG_CRYPTO_SHA512_S390=m -CONFIG_CRYPTO_SHA1_S390=m CONFIG_CRYPTO_SHA3_256_S390=m CONFIG_CRYPTO_SHA3_512_S390=m CONFIG_CRYPTO_GHASH_S390=m diff --git a/arch/s390/crypto/Kconfig b/arch/s390/crypto/Kconfig index e2c27588b21a..03f73fbd38b6 100644 --- a/arch/s390/crypto/Kconfig +++ b/arch/s390/crypto/Kconfig @@ -2,26 +2,6 @@ menu "Accelerated Cryptographic Algorithms for CPU (s390)" -config CRYPTO_SHA512_S390 - tristate "Hash functions: SHA-384 and SHA-512" - select CRYPTO_HASH - help - SHA-384 and SHA-512 secure hash algorithms (FIPS 180) - - Architecture: s390 - - It is available as of z10. - -config CRYPTO_SHA1_S390 - tristate "Hash functions: SHA-1" - select CRYPTO_HASH - help - SHA-1 secure hash algorithm (FIPS 180) - - Architecture: s390 - - It is available as of z990. - config CRYPTO_SHA3_256_S390 tristate "Hash functions: SHA3-224 and SHA3-256" select CRYPTO_HASH diff --git a/arch/s390/crypto/Makefile b/arch/s390/crypto/Makefile index 21757d86cd49..1e5a1038d491 100644 --- a/arch/s390/crypto/Makefile +++ b/arch/s390/crypto/Makefile @@ -3,8 +3,6 @@ # Cryptographic API # -obj-$(CONFIG_CRYPTO_SHA1_S390) += sha1_s390.o sha_common.o -obj-$(CONFIG_CRYPTO_SHA512_S390) += sha512_s390.o sha_common.o obj-$(CONFIG_CRYPTO_SHA3_256_S390) += sha3_256_s390.o sha_common.o obj-$(CONFIG_CRYPTO_SHA3_512_S390) += sha3_512_s390.o sha_common.o obj-$(CONFIG_CRYPTO_DES_S390) += des_s390.o diff --git a/arch/s390/crypto/sha1_s390.c b/arch/s390/crypto/sha1_s390.c deleted file mode 100644 index 9b0d55be1239..000000000000 --- a/arch/s390/crypto/sha1_s390.c +++ /dev/null @@ -1,105 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0+ -/* - * Cryptographic API. - * - * s390 implementation of the SHA1 Secure Hash Algorithm. - * - * Derived from cryptoapi implementation, adapted for in-place - * scatterlist interface. Originally based on the public domain - * implementation written by Steve Reid. - * - * s390 Version: - * Copyright IBM Corp. 2003, 2007 - * Author(s): Thomas Spatzier - * Jan Glauber (jan.glauber@de.ibm.com) - * - * Derived from "crypto/sha1_generic.c" - * Copyright (c) Alan Smithee. - * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk> - * Copyright (c) Jean-Francois Dive <jef@linuxbe.org> - */ -#include <asm/cpacf.h> -#include <crypto/internal/hash.h> -#include <crypto/sha1.h> -#include <linux/cpufeature.h> -#include <linux/kernel.h> -#include <linux/module.h> - -#include "sha.h" - -static int s390_sha1_init(struct shash_desc *desc) -{ - struct s390_sha_ctx *sctx = shash_desc_ctx(desc); - - sctx->state[0] = SHA1_H0; - sctx->state[1] = SHA1_H1; - sctx->state[2] = SHA1_H2; - sctx->state[3] = SHA1_H3; - sctx->state[4] = SHA1_H4; - sctx->count = 0; - sctx->func = CPACF_KIMD_SHA_1; - sctx->first_message_part = 0; - - return 0; -} - -static int s390_sha1_export(struct shash_desc *desc, void *out) -{ - struct s390_sha_ctx *sctx = shash_desc_ctx(desc); - struct sha1_state *octx = out; - - octx->count = sctx->count; - memcpy(octx->state, sctx->state, sizeof(octx->state)); - return 0; -} - -static int s390_sha1_import(struct shash_desc *desc, const void *in) -{ - struct s390_sha_ctx *sctx = shash_desc_ctx(desc); - const struct sha1_state *ictx = in; - - sctx->count = ictx->count; - memcpy(sctx->state, ictx->state, sizeof(ictx->state)); - sctx->func = CPACF_KIMD_SHA_1; - sctx->first_message_part = 0; - return 0; -} - -static struct shash_alg alg = { - .digestsize = SHA1_DIGEST_SIZE, - .init = s390_sha1_init, - .update = s390_sha_update_blocks, - .finup = s390_sha_finup, - .export = s390_sha1_export, - .import = s390_sha1_import, - .descsize = S390_SHA_CTX_SIZE, - .statesize = SHA1_STATE_SIZE, - .base = { - .cra_name = "sha1", - .cra_driver_name= "sha1-s390", - .cra_priority = 300, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SHA1_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}; - -static int __init sha1_s390_init(void) -{ - if (!cpacf_query_func(CPACF_KIMD, CPACF_KIMD_SHA_1)) - return -ENODEV; - return crypto_register_shash(&alg); -} - -static void __exit sha1_s390_fini(void) -{ - crypto_unregister_shash(&alg); -} - -module_cpu_feature_match(S390_CPU_FEATURE_MSA, sha1_s390_init); -module_exit(sha1_s390_fini); - -MODULE_ALIAS_CRYPTO("sha1"); -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm"); diff --git a/arch/s390/crypto/sha512_s390.c b/arch/s390/crypto/sha512_s390.c deleted file mode 100644 index 6cbbf5e8555f..000000000000 --- a/arch/s390/crypto/sha512_s390.c +++ /dev/null @@ -1,154 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0+ -/* - * Cryptographic API. - * - * s390 implementation of the SHA512 and SHA38 Secure Hash Algorithm. - * - * Copyright IBM Corp. 2007 - * Author(s): Jan Glauber (jang@de.ibm.com) - */ -#include <asm/cpacf.h> -#include <crypto/internal/hash.h> -#include <crypto/sha2.h> -#include <linux/cpufeature.h> -#include <linux/errno.h> -#include <linux/kernel.h> -#include <linux/module.h> - -#include "sha.h" - -static int sha512_init(struct shash_desc *desc) -{ - struct s390_sha_ctx *ctx = shash_desc_ctx(desc); - - ctx->sha512.state[0] = SHA512_H0; - ctx->sha512.state[1] = SHA512_H1; - ctx->sha512.state[2] = SHA512_H2; - ctx->sha512.state[3] = SHA512_H3; - ctx->sha512.state[4] = SHA512_H4; - ctx->sha512.state[5] = SHA512_H5; - ctx->sha512.state[6] = SHA512_H6; - ctx->sha512.state[7] = SHA512_H7; - ctx->count = 0; - ctx->sha512.count_hi = 0; - ctx->func = CPACF_KIMD_SHA_512; - ctx->first_message_part = 0; - - return 0; -} - -static int sha512_export(struct shash_desc *desc, void *out) -{ - struct s390_sha_ctx *sctx = shash_desc_ctx(desc); - struct sha512_state *octx = out; - - octx->count[0] = sctx->count; - octx->count[1] = sctx->sha512.count_hi; - memcpy(octx->state, sctx->state, sizeof(octx->state)); - return 0; -} - -static int sha512_import(struct shash_desc *desc, const void *in) -{ - struct s390_sha_ctx *sctx = shash_desc_ctx(desc); - const struct sha512_state *ictx = in; - - sctx->count = ictx->count[0]; - sctx->sha512.count_hi = ictx->count[1]; - - memcpy(sctx->state, ictx->state, sizeof(ictx->state)); - sctx->func = CPACF_KIMD_SHA_512; - sctx->first_message_part = 0; - return 0; -} - -static struct shash_alg sha512_alg = { - .digestsize = SHA512_DIGEST_SIZE, - .init = sha512_init, - .update = s390_sha_update_blocks, - .finup = s390_sha_finup, - .export = sha512_export, - .import = sha512_import, - .descsize = sizeof(struct s390_sha_ctx), - .statesize = SHA512_STATE_SIZE, - .base = { - .cra_name = "sha512", - .cra_driver_name= "sha512-s390", - .cra_priority = 300, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SHA512_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}; - -MODULE_ALIAS_CRYPTO("sha512"); - -static int sha384_init(struct shash_desc *desc) -{ - struct s390_sha_ctx *ctx = shash_desc_ctx(desc); - - ctx->sha512.state[0] = SHA384_H0; - ctx->sha512.state[1] = SHA384_H1; - ctx->sha512.state[2] = SHA384_H2; - ctx->sha512.state[3] = SHA384_H3; - ctx->sha512.state[4] = SHA384_H4; - ctx->sha512.state[5] = SHA384_H5; - ctx->sha512.state[6] = SHA384_H6; - ctx->sha512.state[7] = SHA384_H7; - ctx->count = 0; - ctx->sha512.count_hi = 0; - ctx->func = CPACF_KIMD_SHA_512; - ctx->first_message_part = 0; - - return 0; -} - -static struct shash_alg sha384_alg = { - .digestsize = SHA384_DIGEST_SIZE, - .init = sha384_init, - .update = s390_sha_update_blocks, - .finup = s390_sha_finup, - .export = sha512_export, - .import = sha512_import, - .descsize = sizeof(struct s390_sha_ctx), - .statesize = SHA512_STATE_SIZE, - .base = { - .cra_name = "sha384", - .cra_driver_name= "sha384-s390", - .cra_priority = 300, - .cra_blocksize = SHA384_BLOCK_SIZE, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_ctxsize = sizeof(struct s390_sha_ctx), - .cra_module = THIS_MODULE, - } -}; - -MODULE_ALIAS_CRYPTO("sha384"); - -static int __init init(void) -{ - int ret; - - if (!cpacf_query_func(CPACF_KIMD, CPACF_KIMD_SHA_512)) - return -ENODEV; - if ((ret = crypto_register_shash(&sha512_alg)) < 0) - goto out; - if ((ret = crypto_register_shash(&sha384_alg)) < 0) - crypto_unregister_shash(&sha512_alg); -out: - return ret; -} - -static void __exit fini(void) -{ - crypto_unregister_shash(&sha512_alg); - crypto_unregister_shash(&sha384_alg); -} - -module_cpu_feature_match(S390_CPU_FEATURE_MSA, init); -module_exit(fini); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("SHA512 and SHA-384 Secure Hash Algorithm"); diff --git a/arch/s390/hypfs/hypfs.h b/arch/s390/hypfs/hypfs.h index 83ebf54cca6b..4dc2e068e0ff 100644 --- a/arch/s390/hypfs/hypfs.h +++ b/arch/s390/hypfs/hypfs.h @@ -48,7 +48,7 @@ void hypfs_sprp_exit(void); int __hypfs_fs_init(void); -static inline int hypfs_fs_init(void) +static __always_inline int hypfs_fs_init(void) { if (IS_ENABLED(CONFIG_S390_HYPFS_FS)) return __hypfs_fs_init(); diff --git a/arch/s390/hypfs/hypfs_diag.h b/arch/s390/hypfs/hypfs_diag.h index 7090eff27fef..b5218135b8fe 100644 --- a/arch/s390/hypfs/hypfs_diag.h +++ b/arch/s390/hypfs/hypfs_diag.h @@ -19,7 +19,7 @@ int diag204_store(void *buf, int pages); int __hypfs_diag_fs_init(void); void __hypfs_diag_fs_exit(void); -static inline int hypfs_diag_fs_init(void) +static __always_inline int hypfs_diag_fs_init(void) { if (IS_ENABLED(CONFIG_S390_HYPFS_FS)) return __hypfs_diag_fs_init(); diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S index 0f00f4b06d51..75b0fbb236d0 100644 --- a/arch/s390/kernel/entry.S +++ b/arch/s390/kernel/entry.S @@ -124,7 +124,7 @@ _LPP_OFFSET = __LC_LPP #endif .macro STACKLEAK_ERASE -#ifdef CONFIG_GCC_PLUGIN_STACKLEAK +#ifdef CONFIG_KSTACK_ERASE brasl %r14,stackleak_erase_on_task_stack #endif .endm diff --git a/arch/s390/kernel/ptrace.c b/arch/s390/kernel/ptrace.c index e1240f6b29fa..494216c4b4f3 100644 --- a/arch/s390/kernel/ptrace.c +++ b/arch/s390/kernel/ptrace.c @@ -1209,7 +1209,7 @@ static int s390_runtime_instr_set(struct task_struct *target, static const struct user_regset s390_regsets[] = { { - .core_note_type = NT_PRSTATUS, + USER_REGSET_NOTE_TYPE(PRSTATUS), .n = sizeof(s390_regs) / sizeof(long), .size = sizeof(long), .align = sizeof(long), @@ -1217,7 +1217,7 @@ static const struct user_regset s390_regsets[] = { .set = s390_regs_set, }, { - .core_note_type = NT_PRFPREG, + USER_REGSET_NOTE_TYPE(PRFPREG), .n = sizeof(s390_fp_regs) / sizeof(long), .size = sizeof(long), .align = sizeof(long), @@ -1225,7 +1225,7 @@ static const struct user_regset s390_regsets[] = { .set = s390_fpregs_set, }, { - .core_note_type = NT_S390_SYSTEM_CALL, + USER_REGSET_NOTE_TYPE(S390_SYSTEM_CALL), .n = 1, .size = sizeof(unsigned int), .align = sizeof(unsigned int), @@ -1233,7 +1233,7 @@ static const struct user_regset s390_regsets[] = { .set = s390_system_call_set, }, { - .core_note_type = NT_S390_LAST_BREAK, + USER_REGSET_NOTE_TYPE(S390_LAST_BREAK), .n = 1, .size = sizeof(long), .align = sizeof(long), @@ -1241,7 +1241,7 @@ static const struct user_regset s390_regsets[] = { .set = s390_last_break_set, }, { - .core_note_type = NT_S390_TDB, + USER_REGSET_NOTE_TYPE(S390_TDB), .n = 1, .size = 256, .align = 1, @@ -1249,7 +1249,7 @@ static const struct user_regset s390_regsets[] = { .set = s390_tdb_set, }, { - .core_note_type = NT_S390_VXRS_LOW, + USER_REGSET_NOTE_TYPE(S390_VXRS_LOW), .n = __NUM_VXRS_LOW, .size = sizeof(__u64), .align = sizeof(__u64), @@ -1257,7 +1257,7 @@ static const struct user_regset s390_regsets[] = { .set = s390_vxrs_low_set, }, { - .core_note_type = NT_S390_VXRS_HIGH, + USER_REGSET_NOTE_TYPE(S390_VXRS_HIGH), .n = __NUM_VXRS_HIGH, .size = sizeof(__vector128), .align = sizeof(__vector128), @@ -1265,7 +1265,7 @@ static const struct user_regset s390_regsets[] = { .set = s390_vxrs_high_set, }, { - .core_note_type = NT_S390_GS_CB, + USER_REGSET_NOTE_TYPE(S390_GS_CB), .n = sizeof(struct gs_cb) / sizeof(__u64), .size = sizeof(__u64), .align = sizeof(__u64), @@ -1273,7 +1273,7 @@ static const struct user_regset s390_regsets[] = { .set = s390_gs_cb_set, }, { - .core_note_type = NT_S390_GS_BC, + USER_REGSET_NOTE_TYPE(S390_GS_BC), .n = sizeof(struct gs_cb) / sizeof(__u64), .size = sizeof(__u64), .align = sizeof(__u64), @@ -1281,7 +1281,7 @@ static const struct user_regset s390_regsets[] = { .set = s390_gs_bc_set, }, { - .core_note_type = NT_S390_RI_CB, + USER_REGSET_NOTE_TYPE(S390_RI_CB), .n = sizeof(struct runtime_instr_cb) / sizeof(__u64), .size = sizeof(__u64), .align = sizeof(__u64), @@ -1413,7 +1413,7 @@ static int s390_compat_last_break_set(struct task_struct *target, static const struct user_regset s390_compat_regsets[] = { { - .core_note_type = NT_PRSTATUS, + USER_REGSET_NOTE_TYPE(PRSTATUS), .n = sizeof(s390_compat_regs) / sizeof(compat_long_t), .size = sizeof(compat_long_t), .align = sizeof(compat_long_t), @@ -1421,7 +1421,7 @@ static const struct user_regset s390_compat_regsets[] = { .set = s390_compat_regs_set, }, { - .core_note_type = NT_PRFPREG, + USER_REGSET_NOTE_TYPE(PRFPREG), .n = sizeof(s390_fp_regs) / sizeof(compat_long_t), .size = sizeof(compat_long_t), .align = sizeof(compat_long_t), @@ -1429,7 +1429,7 @@ static const struct user_regset s390_compat_regsets[] = { .set = s390_fpregs_set, }, { - .core_note_type = NT_S390_SYSTEM_CALL, + USER_REGSET_NOTE_TYPE(S390_SYSTEM_CALL), .n = 1, .size = sizeof(compat_uint_t), .align = sizeof(compat_uint_t), @@ -1437,7 +1437,7 @@ static const struct user_regset s390_compat_regsets[] = { .set = s390_system_call_set, }, { - .core_note_type = NT_S390_LAST_BREAK, + USER_REGSET_NOTE_TYPE(S390_LAST_BREAK), .n = 1, .size = sizeof(long), .align = sizeof(long), @@ -1445,7 +1445,7 @@ static const struct user_regset s390_compat_regsets[] = { .set = s390_compat_last_break_set, }, { - .core_note_type = NT_S390_TDB, + USER_REGSET_NOTE_TYPE(S390_TDB), .n = 1, .size = 256, .align = 1, @@ -1453,7 +1453,7 @@ static const struct user_regset s390_compat_regsets[] = { .set = s390_tdb_set, }, { - .core_note_type = NT_S390_VXRS_LOW, + USER_REGSET_NOTE_TYPE(S390_VXRS_LOW), .n = __NUM_VXRS_LOW, .size = sizeof(__u64), .align = sizeof(__u64), @@ -1461,7 +1461,7 @@ static const struct user_regset s390_compat_regsets[] = { .set = s390_vxrs_low_set, }, { - .core_note_type = NT_S390_VXRS_HIGH, + USER_REGSET_NOTE_TYPE(S390_VXRS_HIGH), .n = __NUM_VXRS_HIGH, .size = sizeof(__vector128), .align = sizeof(__vector128), @@ -1469,7 +1469,7 @@ static const struct user_regset s390_compat_regsets[] = { .set = s390_vxrs_high_set, }, { - .core_note_type = NT_S390_HIGH_GPRS, + USER_REGSET_NOTE_TYPE(S390_HIGH_GPRS), .n = sizeof(s390_compat_regs_high) / sizeof(compat_long_t), .size = sizeof(compat_long_t), .align = sizeof(compat_long_t), @@ -1477,7 +1477,7 @@ static const struct user_regset s390_compat_regsets[] = { .set = s390_compat_regs_high_set, }, { - .core_note_type = NT_S390_GS_CB, + USER_REGSET_NOTE_TYPE(S390_GS_CB), .n = sizeof(struct gs_cb) / sizeof(__u64), .size = sizeof(__u64), .align = sizeof(__u64), @@ -1485,7 +1485,7 @@ static const struct user_regset s390_compat_regsets[] = { .set = s390_gs_cb_set, }, { - .core_note_type = NT_S390_GS_BC, + USER_REGSET_NOTE_TYPE(S390_GS_BC), .n = sizeof(struct gs_cb) / sizeof(__u64), .size = sizeof(__u64), .align = sizeof(__u64), @@ -1493,7 +1493,7 @@ static const struct user_regset s390_compat_regsets[] = { .set = s390_gs_bc_set, }, { - .core_note_type = NT_S390_RI_CB, + USER_REGSET_NOTE_TYPE(S390_RI_CB), .n = sizeof(struct runtime_instr_cb) / sizeof(__u64), .size = sizeof(__u64), .align = sizeof(__u64), diff --git a/arch/s390/kernel/syscalls/syscall.tbl b/arch/s390/kernel/syscalls/syscall.tbl index a4569b96ef06..8a6744d658db 100644 --- a/arch/s390/kernel/syscalls/syscall.tbl +++ b/arch/s390/kernel/syscalls/syscall.tbl @@ -470,3 +470,5 @@ 465 common listxattrat sys_listxattrat sys_listxattrat 466 common removexattrat sys_removexattrat sys_removexattrat 467 common open_tree_attr sys_open_tree_attr sys_open_tree_attr +468 common file_getattr sys_file_getattr sys_file_getattr +469 common file_setattr sys_file_setattr sys_file_setattr diff --git a/arch/s390/lib/Makefile b/arch/s390/lib/Makefile index cd35cdbfa871..f43f897d3fc0 100644 --- a/arch/s390/lib/Makefile +++ b/arch/s390/lib/Makefile @@ -3,7 +3,6 @@ # Makefile for s390-specific library files.. # -obj-y += crypto/ lib-y += delay.o string.o uaccess.o find.o spinlock.o tishift.o lib-y += csum-partial.o obj-y += mem.o xor.o @@ -25,6 +24,3 @@ obj-$(CONFIG_S390_MODULES_SANITY_TEST_HELPERS) += test_modules_helpers.o lib-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o obj-$(CONFIG_EXPOLINE_EXTERN) += expoline.o - -obj-$(CONFIG_CRC32_ARCH) += crc32-s390.o -crc32-s390-y := crc32.o crc32le-vx.o crc32be-vx.o diff --git a/arch/s390/lib/crc32-vx.h b/arch/s390/lib/crc32-vx.h deleted file mode 100644 index 652c96e1a822..000000000000 --- a/arch/s390/lib/crc32-vx.h +++ /dev/null @@ -1,12 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ - -#ifndef _CRC32_VX_S390_H -#define _CRC32_VX_S390_H - -#include <linux/types.h> - -u32 crc32_be_vgfm_16(u32 crc, unsigned char const *buf, size_t size); -u32 crc32_le_vgfm_16(u32 crc, unsigned char const *buf, size_t size); -u32 crc32c_le_vgfm_16(u32 crc, unsigned char const *buf, size_t size); - -#endif /* _CRC32_VX_S390_H */ diff --git a/arch/s390/lib/crc32.c b/arch/s390/lib/crc32.c deleted file mode 100644 index 3c4b344417c1..000000000000 --- a/arch/s390/lib/crc32.c +++ /dev/null @@ -1,77 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * CRC-32 implemented with the z/Architecture Vector Extension Facility. - * - * Copyright IBM Corp. 2015 - * Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com> - */ -#define KMSG_COMPONENT "crc32-vx" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt - -#include <linux/module.h> -#include <linux/cpufeature.h> -#include <linux/crc32.h> -#include <asm/fpu.h> -#include "crc32-vx.h" - -#define VX_MIN_LEN 64 -#define VX_ALIGNMENT 16L -#define VX_ALIGN_MASK (VX_ALIGNMENT - 1) - -/* - * DEFINE_CRC32_VX() - Define a CRC-32 function using the vector extension - * - * Creates a function to perform a particular CRC-32 computation. Depending - * on the message buffer, the hardware-accelerated or software implementation - * is used. Note that the message buffer is aligned to improve fetch - * operations of VECTOR LOAD MULTIPLE instructions. - */ -#define DEFINE_CRC32_VX(___fname, ___crc32_vx, ___crc32_sw) \ - u32 ___fname(u32 crc, const u8 *data, size_t datalen) \ - { \ - unsigned long prealign, aligned, remaining; \ - DECLARE_KERNEL_FPU_ONSTACK16(vxstate); \ - \ - if (datalen < VX_MIN_LEN + VX_ALIGN_MASK || !cpu_has_vx()) \ - return ___crc32_sw(crc, data, datalen); \ - \ - if ((unsigned long)data & VX_ALIGN_MASK) { \ - prealign = VX_ALIGNMENT - \ - ((unsigned long)data & VX_ALIGN_MASK); \ - datalen -= prealign; \ - crc = ___crc32_sw(crc, data, prealign); \ - data = (void *)((unsigned long)data + prealign); \ - } \ - \ - aligned = datalen & ~VX_ALIGN_MASK; \ - remaining = datalen & VX_ALIGN_MASK; \ - \ - kernel_fpu_begin(&vxstate, KERNEL_VXR_LOW); \ - crc = ___crc32_vx(crc, data, aligned); \ - kernel_fpu_end(&vxstate, KERNEL_VXR_LOW); \ - \ - if (remaining) \ - crc = ___crc32_sw(crc, data + aligned, remaining); \ - \ - return crc; \ - } \ - EXPORT_SYMBOL(___fname); - -DEFINE_CRC32_VX(crc32_le_arch, crc32_le_vgfm_16, crc32_le_base) -DEFINE_CRC32_VX(crc32_be_arch, crc32_be_vgfm_16, crc32_be_base) -DEFINE_CRC32_VX(crc32c_arch, crc32c_le_vgfm_16, crc32c_base) - -u32 crc32_optimizations(void) -{ - if (cpu_has_vx()) { - return CRC32_LE_OPTIMIZATION | - CRC32_BE_OPTIMIZATION | - CRC32C_OPTIMIZATION; - } - return 0; -} -EXPORT_SYMBOL(crc32_optimizations); - -MODULE_AUTHOR("Hendrik Brueckner <brueckner@linux.vnet.ibm.com>"); -MODULE_DESCRIPTION("CRC-32 algorithms using z/Architecture Vector Extension Facility"); -MODULE_LICENSE("GPL"); diff --git a/arch/s390/lib/crc32be-vx.c b/arch/s390/lib/crc32be-vx.c deleted file mode 100644 index fed7c9c70d05..000000000000 --- a/arch/s390/lib/crc32be-vx.c +++ /dev/null @@ -1,174 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Hardware-accelerated CRC-32 variants for Linux on z Systems - * - * Use the z/Architecture Vector Extension Facility to accelerate the - * computing of CRC-32 checksums. - * - * This CRC-32 implementation algorithm processes the most-significant - * bit first (BE). - * - * Copyright IBM Corp. 2015 - * Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com> - */ - -#include <linux/types.h> -#include <asm/fpu.h> -#include "crc32-vx.h" - -/* Vector register range containing CRC-32 constants */ -#define CONST_R1R2 9 -#define CONST_R3R4 10 -#define CONST_R5 11 -#define CONST_R6 12 -#define CONST_RU_POLY 13 -#define CONST_CRC_POLY 14 - -/* - * The CRC-32 constant block contains reduction constants to fold and - * process particular chunks of the input data stream in parallel. - * - * For the CRC-32 variants, the constants are precomputed according to - * these definitions: - * - * R1 = x4*128+64 mod P(x) - * R2 = x4*128 mod P(x) - * R3 = x128+64 mod P(x) - * R4 = x128 mod P(x) - * R5 = x96 mod P(x) - * R6 = x64 mod P(x) - * - * Barret reduction constant, u, is defined as floor(x**64 / P(x)). - * - * where P(x) is the polynomial in the normal domain and the P'(x) is the - * polynomial in the reversed (bitreflected) domain. - * - * Note that the constant definitions below are extended in order to compute - * intermediate results with a single VECTOR GALOIS FIELD MULTIPLY instruction. - * The rightmost doubleword can be 0 to prevent contribution to the result or - * can be multiplied by 1 to perform an XOR without the need for a separate - * VECTOR EXCLUSIVE OR instruction. - * - * CRC-32 (IEEE 802.3 Ethernet, ...) polynomials: - * - * P(x) = 0x04C11DB7 - * P'(x) = 0xEDB88320 - */ - -static unsigned long constants_CRC_32_BE[] = { - 0x08833794c, 0x0e6228b11, /* R1, R2 */ - 0x0c5b9cd4c, 0x0e8a45605, /* R3, R4 */ - 0x0f200aa66, 1UL << 32, /* R5, x32 */ - 0x0490d678d, 1, /* R6, 1 */ - 0x104d101df, 0, /* u */ - 0x104C11DB7, 0, /* P(x) */ -}; - -/** - * crc32_be_vgfm_16 - Compute CRC-32 (BE variant) with vector registers - * @crc: Initial CRC value, typically ~0. - * @buf: Input buffer pointer, performance might be improved if the - * buffer is on a doubleword boundary. - * @size: Size of the buffer, must be 64 bytes or greater. - * - * Register usage: - * V0: Initial CRC value and intermediate constants and results. - * V1..V4: Data for CRC computation. - * V5..V8: Next data chunks that are fetched from the input buffer. - * V9..V14: CRC-32 constants. - */ -u32 crc32_be_vgfm_16(u32 crc, unsigned char const *buf, size_t size) -{ - /* Load CRC-32 constants */ - fpu_vlm(CONST_R1R2, CONST_CRC_POLY, &constants_CRC_32_BE); - fpu_vzero(0); - - /* Load the initial CRC value into the leftmost word of V0. */ - fpu_vlvgf(0, crc, 0); - - /* Load a 64-byte data chunk and XOR with CRC */ - fpu_vlm(1, 4, buf); - fpu_vx(1, 0, 1); - buf += 64; - size -= 64; - - while (size >= 64) { - /* Load the next 64-byte data chunk into V5 to V8 */ - fpu_vlm(5, 8, buf); - - /* - * Perform a GF(2) multiplication of the doublewords in V1 with - * the reduction constants in V0. The intermediate result is - * then folded (accumulated) with the next data chunk in V5 and - * stored in V1. Repeat this step for the register contents - * in V2, V3, and V4 respectively. - */ - fpu_vgfmag(1, CONST_R1R2, 1, 5); - fpu_vgfmag(2, CONST_R1R2, 2, 6); - fpu_vgfmag(3, CONST_R1R2, 3, 7); - fpu_vgfmag(4, CONST_R1R2, 4, 8); - buf += 64; - size -= 64; - } - - /* Fold V1 to V4 into a single 128-bit value in V1 */ - fpu_vgfmag(1, CONST_R3R4, 1, 2); - fpu_vgfmag(1, CONST_R3R4, 1, 3); - fpu_vgfmag(1, CONST_R3R4, 1, 4); - - while (size >= 16) { - fpu_vl(2, buf); - fpu_vgfmag(1, CONST_R3R4, 1, 2); - buf += 16; - size -= 16; - } - - /* - * The R5 constant is used to fold a 128-bit value into an 96-bit value - * that is XORed with the next 96-bit input data chunk. To use a single - * VGFMG instruction, multiply the rightmost 64-bit with x^32 (1<<32) to - * form an intermediate 96-bit value (with appended zeros) which is then - * XORed with the intermediate reduction result. - */ - fpu_vgfmg(1, CONST_R5, 1); - - /* - * Further reduce the remaining 96-bit value to a 64-bit value using a - * single VGFMG, the rightmost doubleword is multiplied with 0x1. The - * intermediate result is then XORed with the product of the leftmost - * doubleword with R6. The result is a 64-bit value and is subject to - * the Barret reduction. - */ - fpu_vgfmg(1, CONST_R6, 1); - - /* - * The input values to the Barret reduction are the degree-63 polynomial - * in V1 (R(x)), degree-32 generator polynomial, and the reduction - * constant u. The Barret reduction result is the CRC value of R(x) mod - * P(x). - * - * The Barret reduction algorithm is defined as: - * - * 1. T1(x) = floor( R(x) / x^32 ) GF2MUL u - * 2. T2(x) = floor( T1(x) / x^32 ) GF2MUL P(x) - * 3. C(x) = R(x) XOR T2(x) mod x^32 - * - * Note: To compensate the division by x^32, use the vector unpack - * instruction to move the leftmost word into the leftmost doubleword - * of the vector register. The rightmost doubleword is multiplied - * with zero to not contribute to the intermediate results. - */ - - /* T1(x) = floor( R(x) / x^32 ) GF2MUL u */ - fpu_vupllf(2, 1); - fpu_vgfmg(2, CONST_RU_POLY, 2); - - /* - * Compute the GF(2) product of the CRC polynomial in VO with T1(x) in - * V2 and XOR the intermediate result, T2(x), with the value in V1. - * The final result is in the rightmost word of V2. - */ - fpu_vupllf(2, 2); - fpu_vgfmag(2, CONST_CRC_POLY, 2, 1); - return fpu_vlgvf(2, 3); -} diff --git a/arch/s390/lib/crc32le-vx.c b/arch/s390/lib/crc32le-vx.c deleted file mode 100644 index 2f629f394df7..000000000000 --- a/arch/s390/lib/crc32le-vx.c +++ /dev/null @@ -1,240 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Hardware-accelerated CRC-32 variants for Linux on z Systems - * - * Use the z/Architecture Vector Extension Facility to accelerate the - * computing of bitreflected CRC-32 checksums for IEEE 802.3 Ethernet - * and Castagnoli. - * - * This CRC-32 implementation algorithm is bitreflected and processes - * the least-significant bit first (Little-Endian). - * - * Copyright IBM Corp. 2015 - * Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com> - */ - -#include <linux/types.h> -#include <asm/fpu.h> -#include "crc32-vx.h" - -/* Vector register range containing CRC-32 constants */ -#define CONST_PERM_LE2BE 9 -#define CONST_R2R1 10 -#define CONST_R4R3 11 -#define CONST_R5 12 -#define CONST_RU_POLY 13 -#define CONST_CRC_POLY 14 - -/* - * The CRC-32 constant block contains reduction constants to fold and - * process particular chunks of the input data stream in parallel. - * - * For the CRC-32 variants, the constants are precomputed according to - * these definitions: - * - * R1 = [(x4*128+32 mod P'(x) << 32)]' << 1 - * R2 = [(x4*128-32 mod P'(x) << 32)]' << 1 - * R3 = [(x128+32 mod P'(x) << 32)]' << 1 - * R4 = [(x128-32 mod P'(x) << 32)]' << 1 - * R5 = [(x64 mod P'(x) << 32)]' << 1 - * R6 = [(x32 mod P'(x) << 32)]' << 1 - * - * The bitreflected Barret reduction constant, u', is defined as - * the bit reversal of floor(x**64 / P(x)). - * - * where P(x) is the polynomial in the normal domain and the P'(x) is the - * polynomial in the reversed (bitreflected) domain. - * - * CRC-32 (IEEE 802.3 Ethernet, ...) polynomials: - * - * P(x) = 0x04C11DB7 - * P'(x) = 0xEDB88320 - * - * CRC-32C (Castagnoli) polynomials: - * - * P(x) = 0x1EDC6F41 - * P'(x) = 0x82F63B78 - */ - -static unsigned long constants_CRC_32_LE[] = { - 0x0f0e0d0c0b0a0908, 0x0706050403020100, /* BE->LE mask */ - 0x1c6e41596, 0x154442bd4, /* R2, R1 */ - 0x0ccaa009e, 0x1751997d0, /* R4, R3 */ - 0x0, 0x163cd6124, /* R5 */ - 0x0, 0x1f7011641, /* u' */ - 0x0, 0x1db710641 /* P'(x) << 1 */ -}; - -static unsigned long constants_CRC_32C_LE[] = { - 0x0f0e0d0c0b0a0908, 0x0706050403020100, /* BE->LE mask */ - 0x09e4addf8, 0x740eef02, /* R2, R1 */ - 0x14cd00bd6, 0xf20c0dfe, /* R4, R3 */ - 0x0, 0x0dd45aab8, /* R5 */ - 0x0, 0x0dea713f1, /* u' */ - 0x0, 0x105ec76f0 /* P'(x) << 1 */ -}; - -/** - * crc32_le_vgfm_generic - Compute CRC-32 (LE variant) with vector registers - * @crc: Initial CRC value, typically ~0. - * @buf: Input buffer pointer, performance might be improved if the - * buffer is on a doubleword boundary. - * @size: Size of the buffer, must be 64 bytes or greater. - * @constants: CRC-32 constant pool base pointer. - * - * Register usage: - * V0: Initial CRC value and intermediate constants and results. - * V1..V4: Data for CRC computation. - * V5..V8: Next data chunks that are fetched from the input buffer. - * V9: Constant for BE->LE conversion and shift operations - * V10..V14: CRC-32 constants. - */ -static u32 crc32_le_vgfm_generic(u32 crc, unsigned char const *buf, size_t size, unsigned long *constants) -{ - /* Load CRC-32 constants */ - fpu_vlm(CONST_PERM_LE2BE, CONST_CRC_POLY, constants); - - /* - * Load the initial CRC value. - * - * The CRC value is loaded into the rightmost word of the - * vector register and is later XORed with the LSB portion - * of the loaded input data. - */ - fpu_vzero(0); /* Clear V0 */ - fpu_vlvgf(0, crc, 3); /* Load CRC into rightmost word */ - - /* Load a 64-byte data chunk and XOR with CRC */ - fpu_vlm(1, 4, buf); - fpu_vperm(1, 1, 1, CONST_PERM_LE2BE); - fpu_vperm(2, 2, 2, CONST_PERM_LE2BE); - fpu_vperm(3, 3, 3, CONST_PERM_LE2BE); - fpu_vperm(4, 4, 4, CONST_PERM_LE2BE); - - fpu_vx(1, 0, 1); /* V1 ^= CRC */ - buf += 64; - size -= 64; - - while (size >= 64) { - fpu_vlm(5, 8, buf); - fpu_vperm(5, 5, 5, CONST_PERM_LE2BE); - fpu_vperm(6, 6, 6, CONST_PERM_LE2BE); - fpu_vperm(7, 7, 7, CONST_PERM_LE2BE); - fpu_vperm(8, 8, 8, CONST_PERM_LE2BE); - /* - * Perform a GF(2) multiplication of the doublewords in V1 with - * the R1 and R2 reduction constants in V0. The intermediate - * result is then folded (accumulated) with the next data chunk - * in V5 and stored in V1. Repeat this step for the register - * contents in V2, V3, and V4 respectively. - */ - fpu_vgfmag(1, CONST_R2R1, 1, 5); - fpu_vgfmag(2, CONST_R2R1, 2, 6); - fpu_vgfmag(3, CONST_R2R1, 3, 7); - fpu_vgfmag(4, CONST_R2R1, 4, 8); - buf += 64; - size -= 64; - } - - /* - * Fold V1 to V4 into a single 128-bit value in V1. Multiply V1 with R3 - * and R4 and accumulating the next 128-bit chunk until a single 128-bit - * value remains. - */ - fpu_vgfmag(1, CONST_R4R3, 1, 2); - fpu_vgfmag(1, CONST_R4R3, 1, 3); - fpu_vgfmag(1, CONST_R4R3, 1, 4); - - while (size >= 16) { - fpu_vl(2, buf); - fpu_vperm(2, 2, 2, CONST_PERM_LE2BE); - fpu_vgfmag(1, CONST_R4R3, 1, 2); - buf += 16; - size -= 16; - } - - /* - * Set up a vector register for byte shifts. The shift value must - * be loaded in bits 1-4 in byte element 7 of a vector register. - * Shift by 8 bytes: 0x40 - * Shift by 4 bytes: 0x20 - */ - fpu_vleib(9, 0x40, 7); - - /* - * Prepare V0 for the next GF(2) multiplication: shift V0 by 8 bytes - * to move R4 into the rightmost doubleword and set the leftmost - * doubleword to 0x1. - */ - fpu_vsrlb(0, CONST_R4R3, 9); - fpu_vleig(0, 1, 0); - - /* - * Compute GF(2) product of V1 and V0. The rightmost doubleword - * of V1 is multiplied with R4. The leftmost doubleword of V1 is - * multiplied by 0x1 and is then XORed with rightmost product. - * Implicitly, the intermediate leftmost product becomes padded - */ - fpu_vgfmg(1, 0, 1); - - /* - * Now do the final 32-bit fold by multiplying the rightmost word - * in V1 with R5 and XOR the result with the remaining bits in V1. - * - * To achieve this by a single VGFMAG, right shift V1 by a word - * and store the result in V2 which is then accumulated. Use the - * vector unpack instruction to load the rightmost half of the - * doubleword into the rightmost doubleword element of V1; the other - * half is loaded in the leftmost doubleword. - * The vector register with CONST_R5 contains the R5 constant in the - * rightmost doubleword and the leftmost doubleword is zero to ignore - * the leftmost product of V1. - */ - fpu_vleib(9, 0x20, 7); /* Shift by words */ - fpu_vsrlb(2, 1, 9); /* Store remaining bits in V2 */ - fpu_vupllf(1, 1); /* Split rightmost doubleword */ - fpu_vgfmag(1, CONST_R5, 1, 2); /* V1 = (V1 * R5) XOR V2 */ - - /* - * Apply a Barret reduction to compute the final 32-bit CRC value. - * - * The input values to the Barret reduction are the degree-63 polynomial - * in V1 (R(x)), degree-32 generator polynomial, and the reduction - * constant u. The Barret reduction result is the CRC value of R(x) mod - * P(x). - * - * The Barret reduction algorithm is defined as: - * - * 1. T1(x) = floor( R(x) / x^32 ) GF2MUL u - * 2. T2(x) = floor( T1(x) / x^32 ) GF2MUL P(x) - * 3. C(x) = R(x) XOR T2(x) mod x^32 - * - * Note: The leftmost doubleword of vector register containing - * CONST_RU_POLY is zero and, thus, the intermediate GF(2) product - * is zero and does not contribute to the final result. - */ - - /* T1(x) = floor( R(x) / x^32 ) GF2MUL u */ - fpu_vupllf(2, 1); - fpu_vgfmg(2, CONST_RU_POLY, 2); - - /* - * Compute the GF(2) product of the CRC polynomial with T1(x) in - * V2 and XOR the intermediate result, T2(x), with the value in V1. - * The final result is stored in word element 2 of V2. - */ - fpu_vupllf(2, 2); - fpu_vgfmag(2, CONST_CRC_POLY, 2, 1); - - return fpu_vlgvf(2, 2); -} - -u32 crc32_le_vgfm_16(u32 crc, unsigned char const *buf, size_t size) -{ - return crc32_le_vgfm_generic(crc, buf, size, &constants_CRC_32_LE[0]); -} - -u32 crc32c_le_vgfm_16(u32 crc, unsigned char const *buf, size_t size) -{ - return crc32_le_vgfm_generic(crc, buf, size, &constants_CRC_32C_LE[0]); -} diff --git a/arch/s390/lib/crypto/Kconfig b/arch/s390/lib/crypto/Kconfig deleted file mode 100644 index e3f855ef4393..000000000000 --- a/arch/s390/lib/crypto/Kconfig +++ /dev/null @@ -1,13 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only - -config CRYPTO_CHACHA_S390 - tristate - default CRYPTO_LIB_CHACHA - select CRYPTO_LIB_CHACHA_GENERIC - select CRYPTO_ARCH_HAVE_LIB_CHACHA - -config CRYPTO_SHA256_S390 - tristate - default CRYPTO_LIB_SHA256 - select CRYPTO_ARCH_HAVE_LIB_SHA256 - select CRYPTO_LIB_SHA256_GENERIC diff --git a/arch/s390/lib/crypto/Makefile b/arch/s390/lib/crypto/Makefile deleted file mode 100644 index 5df30f1e7930..000000000000 --- a/arch/s390/lib/crypto/Makefile +++ /dev/null @@ -1,7 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only - -obj-$(CONFIG_CRYPTO_CHACHA_S390) += chacha_s390.o -chacha_s390-y := chacha-glue.o chacha-s390.o - -obj-$(CONFIG_CRYPTO_SHA256_S390) += sha256-s390.o -sha256-s390-y := sha256.o diff --git a/arch/s390/lib/crypto/chacha-glue.c b/arch/s390/lib/crypto/chacha-glue.c deleted file mode 100644 index f95ba3483bbc..000000000000 --- a/arch/s390/lib/crypto/chacha-glue.c +++ /dev/null @@ -1,56 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * ChaCha stream cipher (s390 optimized) - * - * Copyright IBM Corp. 2021 - */ - -#define KMSG_COMPONENT "chacha_s390" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt - -#include <crypto/chacha.h> -#include <linux/cpufeature.h> -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/sizes.h> -#include <asm/fpu.h> -#include "chacha-s390.h" - -void hchacha_block_arch(const struct chacha_state *state, - u32 out[HCHACHA_OUT_WORDS], int nrounds) -{ - /* TODO: implement hchacha_block_arch() in assembly */ - hchacha_block_generic(state, out, nrounds); -} -EXPORT_SYMBOL(hchacha_block_arch); - -void chacha_crypt_arch(struct chacha_state *state, u8 *dst, const u8 *src, - unsigned int bytes, int nrounds) -{ - /* s390 chacha20 implementation has 20 rounds hard-coded, - * it cannot handle a block of data or less, but otherwise - * it can handle data of arbitrary size - */ - if (bytes <= CHACHA_BLOCK_SIZE || nrounds != 20 || !cpu_has_vx()) { - chacha_crypt_generic(state, dst, src, bytes, nrounds); - } else { - DECLARE_KERNEL_FPU_ONSTACK32(vxstate); - - kernel_fpu_begin(&vxstate, KERNEL_VXR); - chacha20_vx(dst, src, bytes, &state->x[4], &state->x[12]); - kernel_fpu_end(&vxstate, KERNEL_VXR); - - state->x[12] += round_up(bytes, CHACHA_BLOCK_SIZE) / - CHACHA_BLOCK_SIZE; - } -} -EXPORT_SYMBOL(chacha_crypt_arch); - -bool chacha_is_arch_optimized(void) -{ - return cpu_has_vx(); -} -EXPORT_SYMBOL(chacha_is_arch_optimized); - -MODULE_DESCRIPTION("ChaCha stream cipher (s390 optimized)"); -MODULE_LICENSE("GPL v2"); diff --git a/arch/s390/lib/crypto/chacha-s390.S b/arch/s390/lib/crypto/chacha-s390.S deleted file mode 100644 index 63f3102678c0..000000000000 --- a/arch/s390/lib/crypto/chacha-s390.S +++ /dev/null @@ -1,908 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Original implementation written by Andy Polyakov, @dot-asm. - * This is an adaptation of the original code for kernel use. - * - * Copyright (C) 2006-2019 CRYPTOGAMS by <appro@openssl.org>. All Rights Reserved. - */ - -#include <linux/linkage.h> -#include <asm/nospec-insn.h> -#include <asm/fpu-insn.h> - -#define SP %r15 -#define FRAME (16 * 8 + 4 * 8) - - .data - .balign 32 - -SYM_DATA_START_LOCAL(sigma) - .long 0x61707865,0x3320646e,0x79622d32,0x6b206574 # endian-neutral - .long 1,0,0,0 - .long 2,0,0,0 - .long 3,0,0,0 - .long 0x03020100,0x07060504,0x0b0a0908,0x0f0e0d0c # byte swap - - .long 0,1,2,3 - .long 0x61707865,0x61707865,0x61707865,0x61707865 # smashed sigma - .long 0x3320646e,0x3320646e,0x3320646e,0x3320646e - .long 0x79622d32,0x79622d32,0x79622d32,0x79622d32 - .long 0x6b206574,0x6b206574,0x6b206574,0x6b206574 -SYM_DATA_END(sigma) - - .previous - - GEN_BR_THUNK %r14 - - .text - -############################################################################# -# void chacha20_vx_4x(u8 *out, counst u8 *inp, size_t len, -# counst u32 *key, const u32 *counter) - -#define OUT %r2 -#define INP %r3 -#define LEN %r4 -#define KEY %r5 -#define COUNTER %r6 - -#define BEPERM %v31 -#define CTR %v26 - -#define K0 %v16 -#define K1 %v17 -#define K2 %v18 -#define K3 %v19 - -#define XA0 %v0 -#define XA1 %v1 -#define XA2 %v2 -#define XA3 %v3 - -#define XB0 %v4 -#define XB1 %v5 -#define XB2 %v6 -#define XB3 %v7 - -#define XC0 %v8 -#define XC1 %v9 -#define XC2 %v10 -#define XC3 %v11 - -#define XD0 %v12 -#define XD1 %v13 -#define XD2 %v14 -#define XD3 %v15 - -#define XT0 %v27 -#define XT1 %v28 -#define XT2 %v29 -#define XT3 %v30 - -SYM_FUNC_START(chacha20_vx_4x) - stmg %r6,%r7,6*8(SP) - - larl %r7,sigma - lhi %r0,10 - lhi %r1,0 - - VL K0,0,,%r7 # load sigma - VL K1,0,,KEY # load key - VL K2,16,,KEY - VL K3,0,,COUNTER # load counter - - VL BEPERM,0x40,,%r7 - VL CTR,0x50,,%r7 - - VLM XA0,XA3,0x60,%r7,4 # load [smashed] sigma - - VREPF XB0,K1,0 # smash the key - VREPF XB1,K1,1 - VREPF XB2,K1,2 - VREPF XB3,K1,3 - - VREPF XD0,K3,0 - VREPF XD1,K3,1 - VREPF XD2,K3,2 - VREPF XD3,K3,3 - VAF XD0,XD0,CTR - - VREPF XC0,K2,0 - VREPF XC1,K2,1 - VREPF XC2,K2,2 - VREPF XC3,K2,3 - -.Loop_4x: - VAF XA0,XA0,XB0 - VX XD0,XD0,XA0 - VERLLF XD0,XD0,16 - - VAF XA1,XA1,XB1 - VX XD1,XD1,XA1 - VERLLF XD1,XD1,16 - - VAF XA2,XA2,XB2 - VX XD2,XD2,XA2 - VERLLF XD2,XD2,16 - - VAF XA3,XA3,XB3 - VX XD3,XD3,XA3 - VERLLF XD3,XD3,16 - - VAF XC0,XC0,XD0 - VX XB0,XB0,XC0 - VERLLF XB0,XB0,12 - - VAF XC1,XC1,XD1 - VX XB1,XB1,XC1 - VERLLF XB1,XB1,12 - - VAF XC2,XC2,XD2 - VX XB2,XB2,XC2 - VERLLF XB2,XB2,12 - - VAF XC3,XC3,XD3 - VX XB3,XB3,XC3 - VERLLF XB3,XB3,12 - - VAF XA0,XA0,XB0 - VX XD0,XD0,XA0 - VERLLF XD0,XD0,8 - - VAF XA1,XA1,XB1 - VX XD1,XD1,XA1 - VERLLF XD1,XD1,8 - - VAF XA2,XA2,XB2 - VX XD2,XD2,XA2 - VERLLF XD2,XD2,8 - - VAF XA3,XA3,XB3 - VX XD3,XD3,XA3 - VERLLF XD3,XD3,8 - - VAF XC0,XC0,XD0 - VX XB0,XB0,XC0 - VERLLF XB0,XB0,7 - - VAF XC1,XC1,XD1 - VX XB1,XB1,XC1 - VERLLF XB1,XB1,7 - - VAF XC2,XC2,XD2 - VX XB2,XB2,XC2 - VERLLF XB2,XB2,7 - - VAF XC3,XC3,XD3 - VX XB3,XB3,XC3 - VERLLF XB3,XB3,7 - - VAF XA0,XA0,XB1 - VX XD3,XD3,XA0 - VERLLF XD3,XD3,16 - - VAF XA1,XA1,XB2 - VX XD0,XD0,XA1 - VERLLF XD0,XD0,16 - - VAF XA2,XA2,XB3 - VX XD1,XD1,XA2 - VERLLF XD1,XD1,16 - - VAF XA3,XA3,XB0 - VX XD2,XD2,XA3 - VERLLF XD2,XD2,16 - - VAF XC2,XC2,XD3 - VX XB1,XB1,XC2 - VERLLF XB1,XB1,12 - - VAF XC3,XC3,XD0 - VX XB2,XB2,XC3 - VERLLF XB2,XB2,12 - - VAF XC0,XC0,XD1 - VX XB3,XB3,XC0 - VERLLF XB3,XB3,12 - - VAF XC1,XC1,XD2 - VX XB0,XB0,XC1 - VERLLF XB0,XB0,12 - - VAF XA0,XA0,XB1 - VX XD3,XD3,XA0 - VERLLF XD3,XD3,8 - - VAF XA1,XA1,XB2 - VX XD0,XD0,XA1 - VERLLF XD0,XD0,8 - - VAF XA2,XA2,XB3 - VX XD1,XD1,XA2 - VERLLF XD1,XD1,8 - - VAF XA3,XA3,XB0 - VX XD2,XD2,XA3 - VERLLF XD2,XD2,8 - - VAF XC2,XC2,XD3 - VX XB1,XB1,XC2 - VERLLF XB1,XB1,7 - - VAF XC3,XC3,XD0 - VX XB2,XB2,XC3 - VERLLF XB2,XB2,7 - - VAF XC0,XC0,XD1 - VX XB3,XB3,XC0 - VERLLF XB3,XB3,7 - - VAF XC1,XC1,XD2 - VX XB0,XB0,XC1 - VERLLF XB0,XB0,7 - brct %r0,.Loop_4x - - VAF XD0,XD0,CTR - - VMRHF XT0,XA0,XA1 # transpose data - VMRHF XT1,XA2,XA3 - VMRLF XT2,XA0,XA1 - VMRLF XT3,XA2,XA3 - VPDI XA0,XT0,XT1,0b0000 - VPDI XA1,XT0,XT1,0b0101 - VPDI XA2,XT2,XT3,0b0000 - VPDI XA3,XT2,XT3,0b0101 - - VMRHF XT0,XB0,XB1 - VMRHF XT1,XB2,XB3 - VMRLF XT2,XB0,XB1 - VMRLF XT3,XB2,XB3 - VPDI XB0,XT0,XT1,0b0000 - VPDI XB1,XT0,XT1,0b0101 - VPDI XB2,XT2,XT3,0b0000 - VPDI XB3,XT2,XT3,0b0101 - - VMRHF XT0,XC0,XC1 - VMRHF XT1,XC2,XC3 - VMRLF XT2,XC0,XC1 - VMRLF XT3,XC2,XC3 - VPDI XC0,XT0,XT1,0b0000 - VPDI XC1,XT0,XT1,0b0101 - VPDI XC2,XT2,XT3,0b0000 - VPDI XC3,XT2,XT3,0b0101 - - VMRHF XT0,XD0,XD1 - VMRHF XT1,XD2,XD3 - VMRLF XT2,XD0,XD1 - VMRLF XT3,XD2,XD3 - VPDI XD0,XT0,XT1,0b0000 - VPDI XD1,XT0,XT1,0b0101 - VPDI XD2,XT2,XT3,0b0000 - VPDI XD3,XT2,XT3,0b0101 - - VAF XA0,XA0,K0 - VAF XB0,XB0,K1 - VAF XC0,XC0,K2 - VAF XD0,XD0,K3 - - VPERM XA0,XA0,XA0,BEPERM - VPERM XB0,XB0,XB0,BEPERM - VPERM XC0,XC0,XC0,BEPERM - VPERM XD0,XD0,XD0,BEPERM - - VLM XT0,XT3,0,INP,0 - - VX XT0,XT0,XA0 - VX XT1,XT1,XB0 - VX XT2,XT2,XC0 - VX XT3,XT3,XD0 - - VSTM XT0,XT3,0,OUT,0 - - la INP,0x40(INP) - la OUT,0x40(OUT) - aghi LEN,-0x40 - - VAF XA0,XA1,K0 - VAF XB0,XB1,K1 - VAF XC0,XC1,K2 - VAF XD0,XD1,K3 - - VPERM XA0,XA0,XA0,BEPERM - VPERM XB0,XB0,XB0,BEPERM - VPERM XC0,XC0,XC0,BEPERM - VPERM XD0,XD0,XD0,BEPERM - - clgfi LEN,0x40 - jl .Ltail_4x - - VLM XT0,XT3,0,INP,0 - - VX XT0,XT0,XA0 - VX XT1,XT1,XB0 - VX XT2,XT2,XC0 - VX XT3,XT3,XD0 - - VSTM XT0,XT3,0,OUT,0 - - la INP,0x40(INP) - la OUT,0x40(OUT) - aghi LEN,-0x40 - je .Ldone_4x - - VAF XA0,XA2,K0 - VAF XB0,XB2,K1 - VAF XC0,XC2,K2 - VAF XD0,XD2,K3 - - VPERM XA0,XA0,XA0,BEPERM - VPERM XB0,XB0,XB0,BEPERM - VPERM XC0,XC0,XC0,BEPERM - VPERM XD0,XD0,XD0,BEPERM - - clgfi LEN,0x40 - jl .Ltail_4x - - VLM XT0,XT3,0,INP,0 - - VX XT0,XT0,XA0 - VX XT1,XT1,XB0 - VX XT2,XT2,XC0 - VX XT3,XT3,XD0 - - VSTM XT0,XT3,0,OUT,0 - - la INP,0x40(INP) - la OUT,0x40(OUT) - aghi LEN,-0x40 - je .Ldone_4x - - VAF XA0,XA3,K0 - VAF XB0,XB3,K1 - VAF XC0,XC3,K2 - VAF XD0,XD3,K3 - - VPERM XA0,XA0,XA0,BEPERM - VPERM XB0,XB0,XB0,BEPERM - VPERM XC0,XC0,XC0,BEPERM - VPERM XD0,XD0,XD0,BEPERM - - clgfi LEN,0x40 - jl .Ltail_4x - - VLM XT0,XT3,0,INP,0 - - VX XT0,XT0,XA0 - VX XT1,XT1,XB0 - VX XT2,XT2,XC0 - VX XT3,XT3,XD0 - - VSTM XT0,XT3,0,OUT,0 - -.Ldone_4x: - lmg %r6,%r7,6*8(SP) - BR_EX %r14 - -.Ltail_4x: - VLR XT0,XC0 - VLR XT1,XD0 - - VST XA0,8*8+0x00,,SP - VST XB0,8*8+0x10,,SP - VST XT0,8*8+0x20,,SP - VST XT1,8*8+0x30,,SP - - lghi %r1,0 - -.Loop_tail_4x: - llgc %r5,0(%r1,INP) - llgc %r6,8*8(%r1,SP) - xr %r6,%r5 - stc %r6,0(%r1,OUT) - la %r1,1(%r1) - brct LEN,.Loop_tail_4x - - lmg %r6,%r7,6*8(SP) - BR_EX %r14 -SYM_FUNC_END(chacha20_vx_4x) - -#undef OUT -#undef INP -#undef LEN -#undef KEY -#undef COUNTER - -#undef BEPERM - -#undef K0 -#undef K1 -#undef K2 -#undef K3 - - -############################################################################# -# void chacha20_vx(u8 *out, counst u8 *inp, size_t len, -# counst u32 *key, const u32 *counter) - -#define OUT %r2 -#define INP %r3 -#define LEN %r4 -#define KEY %r5 -#define COUNTER %r6 - -#define BEPERM %v31 - -#define K0 %v27 -#define K1 %v24 -#define K2 %v25 -#define K3 %v26 - -#define A0 %v0 -#define B0 %v1 -#define C0 %v2 -#define D0 %v3 - -#define A1 %v4 -#define B1 %v5 -#define C1 %v6 -#define D1 %v7 - -#define A2 %v8 -#define B2 %v9 -#define C2 %v10 -#define D2 %v11 - -#define A3 %v12 -#define B3 %v13 -#define C3 %v14 -#define D3 %v15 - -#define A4 %v16 -#define B4 %v17 -#define C4 %v18 -#define D4 %v19 - -#define A5 %v20 -#define B5 %v21 -#define C5 %v22 -#define D5 %v23 - -#define T0 %v27 -#define T1 %v28 -#define T2 %v29 -#define T3 %v30 - -SYM_FUNC_START(chacha20_vx) - clgfi LEN,256 - jle chacha20_vx_4x - stmg %r6,%r7,6*8(SP) - - lghi %r1,-FRAME - lgr %r0,SP - la SP,0(%r1,SP) - stg %r0,0(SP) # back-chain - - larl %r7,sigma - lhi %r0,10 - - VLM K1,K2,0,KEY,0 # load key - VL K3,0,,COUNTER # load counter - - VLM K0,BEPERM,0,%r7,4 # load sigma, increments, ... - -.Loop_outer_vx: - VLR A0,K0 - VLR B0,K1 - VLR A1,K0 - VLR B1,K1 - VLR A2,K0 - VLR B2,K1 - VLR A3,K0 - VLR B3,K1 - VLR A4,K0 - VLR B4,K1 - VLR A5,K0 - VLR B5,K1 - - VLR D0,K3 - VAF D1,K3,T1 # K[3]+1 - VAF D2,K3,T2 # K[3]+2 - VAF D3,K3,T3 # K[3]+3 - VAF D4,D2,T2 # K[3]+4 - VAF D5,D2,T3 # K[3]+5 - - VLR C0,K2 - VLR C1,K2 - VLR C2,K2 - VLR C3,K2 - VLR C4,K2 - VLR C5,K2 - - VLR T1,D1 - VLR T2,D2 - VLR T3,D3 - -.Loop_vx: - VAF A0,A0,B0 - VAF A1,A1,B1 - VAF A2,A2,B2 - VAF A3,A3,B3 - VAF A4,A4,B4 - VAF A5,A5,B5 - VX D0,D0,A0 - VX D1,D1,A1 - VX D2,D2,A2 - VX D3,D3,A3 - VX D4,D4,A4 - VX D5,D5,A5 - VERLLF D0,D0,16 - VERLLF D1,D1,16 - VERLLF D2,D2,16 - VERLLF D3,D3,16 - VERLLF D4,D4,16 - VERLLF D5,D5,16 - - VAF C0,C0,D0 - VAF C1,C1,D1 - VAF C2,C2,D2 - VAF C3,C3,D3 - VAF C4,C4,D4 - VAF C5,C5,D5 - VX B0,B0,C0 - VX B1,B1,C1 - VX B2,B2,C2 - VX B3,B3,C3 - VX B4,B4,C4 - VX B5,B5,C5 - VERLLF B0,B0,12 - VERLLF B1,B1,12 - VERLLF B2,B2,12 - VERLLF B3,B3,12 - VERLLF B4,B4,12 - VERLLF B5,B5,12 - - VAF A0,A0,B0 - VAF A1,A1,B1 - VAF A2,A2,B2 - VAF A3,A3,B3 - VAF A4,A4,B4 - VAF A5,A5,B5 - VX D0,D0,A0 - VX D1,D1,A1 - VX D2,D2,A2 - VX D3,D3,A3 - VX D4,D4,A4 - VX D5,D5,A5 - VERLLF D0,D0,8 - VERLLF D1,D1,8 - VERLLF D2,D2,8 - VERLLF D3,D3,8 - VERLLF D4,D4,8 - VERLLF D5,D5,8 - - VAF C0,C0,D0 - VAF C1,C1,D1 - VAF C2,C2,D2 - VAF C3,C3,D3 - VAF C4,C4,D4 - VAF C5,C5,D5 - VX B0,B0,C0 - VX B1,B1,C1 - VX B2,B2,C2 - VX B3,B3,C3 - VX B4,B4,C4 - VX B5,B5,C5 - VERLLF B0,B0,7 - VERLLF B1,B1,7 - VERLLF B2,B2,7 - VERLLF B3,B3,7 - VERLLF B4,B4,7 - VERLLF B5,B5,7 - - VSLDB C0,C0,C0,8 - VSLDB C1,C1,C1,8 - VSLDB C2,C2,C2,8 - VSLDB C3,C3,C3,8 - VSLDB C4,C4,C4,8 - VSLDB C5,C5,C5,8 - VSLDB B0,B0,B0,4 - VSLDB B1,B1,B1,4 - VSLDB B2,B2,B2,4 - VSLDB B3,B3,B3,4 - VSLDB B4,B4,B4,4 - VSLDB B5,B5,B5,4 - VSLDB D0,D0,D0,12 - VSLDB D1,D1,D1,12 - VSLDB D2,D2,D2,12 - VSLDB D3,D3,D3,12 - VSLDB D4,D4,D4,12 - VSLDB D5,D5,D5,12 - - VAF A0,A0,B0 - VAF A1,A1,B1 - VAF A2,A2,B2 - VAF A3,A3,B3 - VAF A4,A4,B4 - VAF A5,A5,B5 - VX D0,D0,A0 - VX D1,D1,A1 - VX D2,D2,A2 - VX D3,D3,A3 - VX D4,D4,A4 - VX D5,D5,A5 - VERLLF D0,D0,16 - VERLLF D1,D1,16 - VERLLF D2,D2,16 - VERLLF D3,D3,16 - VERLLF D4,D4,16 - VERLLF D5,D5,16 - - VAF C0,C0,D0 - VAF C1,C1,D1 - VAF C2,C2,D2 - VAF C3,C3,D3 - VAF C4,C4,D4 - VAF C5,C5,D5 - VX B0,B0,C0 - VX B1,B1,C1 - VX B2,B2,C2 - VX B3,B3,C3 - VX B4,B4,C4 - VX B5,B5,C5 - VERLLF B0,B0,12 - VERLLF B1,B1,12 - VERLLF B2,B2,12 - VERLLF B3,B3,12 - VERLLF B4,B4,12 - VERLLF B5,B5,12 - - VAF A0,A0,B0 - VAF A1,A1,B1 - VAF A2,A2,B2 - VAF A3,A3,B3 - VAF A4,A4,B4 - VAF A5,A5,B5 - VX D0,D0,A0 - VX D1,D1,A1 - VX D2,D2,A2 - VX D3,D3,A3 - VX D4,D4,A4 - VX D5,D5,A5 - VERLLF D0,D0,8 - VERLLF D1,D1,8 - VERLLF D2,D2,8 - VERLLF D3,D3,8 - VERLLF D4,D4,8 - VERLLF D5,D5,8 - - VAF C0,C0,D0 - VAF C1,C1,D1 - VAF C2,C2,D2 - VAF C3,C3,D3 - VAF C4,C4,D4 - VAF C5,C5,D5 - VX B0,B0,C0 - VX B1,B1,C1 - VX B2,B2,C2 - VX B3,B3,C3 - VX B4,B4,C4 - VX B5,B5,C5 - VERLLF B0,B0,7 - VERLLF B1,B1,7 - VERLLF B2,B2,7 - VERLLF B3,B3,7 - VERLLF B4,B4,7 - VERLLF B5,B5,7 - - VSLDB C0,C0,C0,8 - VSLDB C1,C1,C1,8 - VSLDB C2,C2,C2,8 - VSLDB C3,C3,C3,8 - VSLDB C4,C4,C4,8 - VSLDB C5,C5,C5,8 - VSLDB B0,B0,B0,12 - VSLDB B1,B1,B1,12 - VSLDB B2,B2,B2,12 - VSLDB B3,B3,B3,12 - VSLDB B4,B4,B4,12 - VSLDB B5,B5,B5,12 - VSLDB D0,D0,D0,4 - VSLDB D1,D1,D1,4 - VSLDB D2,D2,D2,4 - VSLDB D3,D3,D3,4 - VSLDB D4,D4,D4,4 - VSLDB D5,D5,D5,4 - brct %r0,.Loop_vx - - VAF A0,A0,K0 - VAF B0,B0,K1 - VAF C0,C0,K2 - VAF D0,D0,K3 - VAF A1,A1,K0 - VAF D1,D1,T1 # +K[3]+1 - - VPERM A0,A0,A0,BEPERM - VPERM B0,B0,B0,BEPERM - VPERM C0,C0,C0,BEPERM - VPERM D0,D0,D0,BEPERM - - clgfi LEN,0x40 - jl .Ltail_vx - - VAF D2,D2,T2 # +K[3]+2 - VAF D3,D3,T3 # +K[3]+3 - VLM T0,T3,0,INP,0 - - VX A0,A0,T0 - VX B0,B0,T1 - VX C0,C0,T2 - VX D0,D0,T3 - - VLM K0,T3,0,%r7,4 # re-load sigma and increments - - VSTM A0,D0,0,OUT,0 - - la INP,0x40(INP) - la OUT,0x40(OUT) - aghi LEN,-0x40 - je .Ldone_vx - - VAF B1,B1,K1 - VAF C1,C1,K2 - - VPERM A0,A1,A1,BEPERM - VPERM B0,B1,B1,BEPERM - VPERM C0,C1,C1,BEPERM - VPERM D0,D1,D1,BEPERM - - clgfi LEN,0x40 - jl .Ltail_vx - - VLM A1,D1,0,INP,0 - - VX A0,A0,A1 - VX B0,B0,B1 - VX C0,C0,C1 - VX D0,D0,D1 - - VSTM A0,D0,0,OUT,0 - - la INP,0x40(INP) - la OUT,0x40(OUT) - aghi LEN,-0x40 - je .Ldone_vx - - VAF A2,A2,K0 - VAF B2,B2,K1 - VAF C2,C2,K2 - - VPERM A0,A2,A2,BEPERM - VPERM B0,B2,B2,BEPERM - VPERM C0,C2,C2,BEPERM - VPERM D0,D2,D2,BEPERM - - clgfi LEN,0x40 - jl .Ltail_vx - - VLM A1,D1,0,INP,0 - - VX A0,A0,A1 - VX B0,B0,B1 - VX C0,C0,C1 - VX D0,D0,D1 - - VSTM A0,D0,0,OUT,0 - - la INP,0x40(INP) - la OUT,0x40(OUT) - aghi LEN,-0x40 - je .Ldone_vx - - VAF A3,A3,K0 - VAF B3,B3,K1 - VAF C3,C3,K2 - VAF D2,K3,T3 # K[3]+3 - - VPERM A0,A3,A3,BEPERM - VPERM B0,B3,B3,BEPERM - VPERM C0,C3,C3,BEPERM - VPERM D0,D3,D3,BEPERM - - clgfi LEN,0x40 - jl .Ltail_vx - - VAF D3,D2,T1 # K[3]+4 - VLM A1,D1,0,INP,0 - - VX A0,A0,A1 - VX B0,B0,B1 - VX C0,C0,C1 - VX D0,D0,D1 - - VSTM A0,D0,0,OUT,0 - - la INP,0x40(INP) - la OUT,0x40(OUT) - aghi LEN,-0x40 - je .Ldone_vx - - VAF A4,A4,K0 - VAF B4,B4,K1 - VAF C4,C4,K2 - VAF D4,D4,D3 # +K[3]+4 - VAF D3,D3,T1 # K[3]+5 - VAF K3,D2,T3 # K[3]+=6 - - VPERM A0,A4,A4,BEPERM - VPERM B0,B4,B4,BEPERM - VPERM C0,C4,C4,BEPERM - VPERM D0,D4,D4,BEPERM - - clgfi LEN,0x40 - jl .Ltail_vx - - VLM A1,D1,0,INP,0 - - VX A0,A0,A1 - VX B0,B0,B1 - VX C0,C0,C1 - VX D0,D0,D1 - - VSTM A0,D0,0,OUT,0 - - la INP,0x40(INP) - la OUT,0x40(OUT) - aghi LEN,-0x40 - je .Ldone_vx - - VAF A5,A5,K0 - VAF B5,B5,K1 - VAF C5,C5,K2 - VAF D5,D5,D3 # +K[3]+5 - - VPERM A0,A5,A5,BEPERM - VPERM B0,B5,B5,BEPERM - VPERM C0,C5,C5,BEPERM - VPERM D0,D5,D5,BEPERM - - clgfi LEN,0x40 - jl .Ltail_vx - - VLM A1,D1,0,INP,0 - - VX A0,A0,A1 - VX B0,B0,B1 - VX C0,C0,C1 - VX D0,D0,D1 - - VSTM A0,D0,0,OUT,0 - - la INP,0x40(INP) - la OUT,0x40(OUT) - lhi %r0,10 - aghi LEN,-0x40 - jne .Loop_outer_vx - -.Ldone_vx: - lmg %r6,%r7,FRAME+6*8(SP) - la SP,FRAME(SP) - BR_EX %r14 - -.Ltail_vx: - VSTM A0,D0,8*8,SP,3 - lghi %r1,0 - -.Loop_tail_vx: - llgc %r5,0(%r1,INP) - llgc %r6,8*8(%r1,SP) - xr %r6,%r5 - stc %r6,0(%r1,OUT) - la %r1,1(%r1) - brct LEN,.Loop_tail_vx - - lmg %r6,%r7,FRAME+6*8(SP) - la SP,FRAME(SP) - BR_EX %r14 -SYM_FUNC_END(chacha20_vx) - -.previous diff --git a/arch/s390/lib/crypto/chacha-s390.h b/arch/s390/lib/crypto/chacha-s390.h deleted file mode 100644 index 733744ce30f5..000000000000 --- a/arch/s390/lib/crypto/chacha-s390.h +++ /dev/null @@ -1,14 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * s390 ChaCha stream cipher. - * - * Copyright IBM Corp. 2021 - */ - -#ifndef _CHACHA_S390_H -#define _CHACHA_S390_H - -void chacha20_vx(u8 *out, const u8 *inp, size_t len, const u32 *key, - const u32 *counter); - -#endif /* _CHACHA_S390_H */ diff --git a/arch/s390/lib/crypto/sha256.c b/arch/s390/lib/crypto/sha256.c deleted file mode 100644 index 7dfe120fafab..000000000000 --- a/arch/s390/lib/crypto/sha256.c +++ /dev/null @@ -1,47 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * SHA-256 optimized using the CP Assist for Cryptographic Functions (CPACF) - * - * Copyright 2025 Google LLC - */ -#include <asm/cpacf.h> -#include <crypto/internal/sha2.h> -#include <linux/cpufeature.h> -#include <linux/kernel.h> -#include <linux/module.h> - -static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_cpacf_sha256); - -void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], - const u8 *data, size_t nblocks) -{ - if (static_branch_likely(&have_cpacf_sha256)) - cpacf_kimd(CPACF_KIMD_SHA_256, state, data, - nblocks * SHA256_BLOCK_SIZE); - else - sha256_blocks_generic(state, data, nblocks); -} -EXPORT_SYMBOL_GPL(sha256_blocks_arch); - -bool sha256_is_arch_optimized(void) -{ - return static_key_enabled(&have_cpacf_sha256); -} -EXPORT_SYMBOL_GPL(sha256_is_arch_optimized); - -static int __init sha256_s390_mod_init(void) -{ - if (cpu_have_feature(S390_CPU_FEATURE_MSA) && - cpacf_query_func(CPACF_KIMD, CPACF_KIMD_SHA_256)) - static_branch_enable(&have_cpacf_sha256); - return 0; -} -subsys_initcall(sha256_s390_mod_init); - -static void __exit sha256_s390_mod_exit(void) -{ -} -module_exit(sha256_s390_mod_exit); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("SHA-256 using the CP Assist for Cryptographic Functions (CPACF)"); diff --git a/arch/s390/mm/init.c b/arch/s390/mm/init.c index 074bf4fb4ce2..e4953453d254 100644 --- a/arch/s390/mm/init.c +++ b/arch/s390/mm/init.c @@ -142,7 +142,7 @@ bool force_dma_unencrypted(struct device *dev) } /* protected virtualization */ -static void pv_init(void) +static void __init pv_init(void) { if (!is_prot_virt_guest()) return; diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c index c7f8313ba449..0c9a35782c83 100644 --- a/arch/s390/net/bpf_jit_comp.c +++ b/arch/s390/net/bpf_jit_comp.c @@ -566,7 +566,15 @@ static void bpf_jit_plt(struct bpf_plt *plt, void *ret, void *target) { memcpy(plt, &bpf_plt, sizeof(*plt)); plt->ret = ret; - plt->target = target; + /* + * (target == NULL) implies that the branch to this PLT entry was + * patched and became a no-op. However, some CPU could have jumped + * to this PLT entry before patching and may be still executing it. + * + * Since the intention in this case is to make the PLT entry a no-op, + * make the target point to the return label instead of NULL. + */ + plt->target = target ?: ret; } /* diff --git a/arch/s390/purgatory/purgatory.c b/arch/s390/purgatory/purgatory.c index 030efda05dbe..ecb38102187c 100644 --- a/arch/s390/purgatory/purgatory.c +++ b/arch/s390/purgatory/purgatory.c @@ -16,7 +16,7 @@ int verify_sha256_digest(void) { struct kexec_sha_region *ptr, *end; u8 digest[SHA256_DIGEST_SIZE]; - struct sha256_state sctx; + struct sha256_ctx sctx; sha256_init(&sctx); end = purgatory_sha_regions + ARRAY_SIZE(purgatory_sha_regions); |