diff options
115 files changed, 3246 insertions, 1916 deletions
diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index fe565f3a3a91..2dc722fb31b4 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -127,6 +127,7 @@ config S390 select ARCH_WANT_DEFAULT_BPF_JIT select ARCH_WANT_IPC_PARSE_VERSION select ARCH_WANT_KERNEL_PMD_MKWRITE + select ARCH_WANT_LD_ORPHAN_WARN select ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP select BUILDTIME_TABLE_SORT select CLONE_BACKWARDS2 @@ -448,7 +449,7 @@ config COMPAT select COMPAT_OLD_SIGACTION select HAVE_UID16 depends on MULTIUSER - depends on !CC_IS_CLANG + depends on !CC_IS_CLANG && !LD_IS_LLD help Select this option if you want to enable your system kernel to handle system-calls from ELF binaries for 31 bit ESA. This option @@ -582,14 +583,23 @@ config RELOCATABLE help This builds a kernel image that retains relocation information so it can be loaded at an arbitrary address. - The kernel is linked as a position-independent executable (PIE) - and contains dynamic relocations which are processed early in the - bootup process. The relocations make the kernel image about 15% larger (compressed 10%), but are discarded at runtime. Note: this option exists only for documentation purposes, please do not remove it. +config PIE_BUILD + def_bool CC_IS_CLANG && !$(cc-option,-munaligned-symbols) + help + If the compiler is unable to generate code that can manage unaligned + symbols, the kernel is linked as a position-independent executable + (PIE) and includes dynamic relocations that are processed early + during bootup. + + For kpatch functionality, it is recommended to build the kernel + without the PIE_BUILD option. PIE_BUILD is only enabled when the + compiler lacks proper support for handling unaligned symbols. + config RANDOMIZE_BASE bool "Randomize the address of the kernel image (KASLR)" default y diff --git a/arch/s390/Makefile b/arch/s390/Makefile index 73873e451686..2a58e1864931 100644 --- a/arch/s390/Makefile +++ b/arch/s390/Makefile @@ -14,8 +14,14 @@ KBUILD_AFLAGS_MODULE += -fPIC KBUILD_CFLAGS_MODULE += -fPIC KBUILD_AFLAGS += -m64 KBUILD_CFLAGS += -m64 +ifdef CONFIG_PIE_BUILD KBUILD_CFLAGS += -fPIE -LDFLAGS_vmlinux := -pie +LDFLAGS_vmlinux := -pie -z notext +else +KBUILD_CFLAGS += $(call cc-option,-munaligned-symbols,) +LDFLAGS_vmlinux := --emit-relocs --discard-none +extra_tools := relocs +endif aflags_dwarf := -Wa,-gdwarf-2 KBUILD_AFLAGS_DECOMPRESSOR := $(CLANG_FLAGS) -m64 -D__ASSEMBLY__ ifndef CONFIG_AS_IS_LLVM @@ -143,7 +149,7 @@ archheaders: archprepare: $(Q)$(MAKE) $(build)=$(syscalls) kapi - $(Q)$(MAKE) $(build)=$(tools) kapi + $(Q)$(MAKE) $(build)=$(tools) kapi $(extra_tools) ifeq ($(KBUILD_EXTMOD),) # We need to generate vdso-offsets.h before compiling certain files in kernel/. # In order to do that, we should use the archprepare target, but we can't since diff --git a/arch/s390/boot/.gitignore b/arch/s390/boot/.gitignore index f56591bc0897..f5ef099e2fd3 100644 --- a/arch/s390/boot/.gitignore +++ b/arch/s390/boot/.gitignore @@ -1,6 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only image bzImage +relocs.S section_cmp.* vmlinux vmlinux.lds diff --git a/arch/s390/boot/Makefile b/arch/s390/boot/Makefile index c7c81e5f9218..294f08a8811a 100644 --- a/arch/s390/boot/Makefile +++ b/arch/s390/boot/Makefile @@ -37,7 +37,8 @@ CFLAGS_sclp_early_core.o += -I$(srctree)/drivers/s390/char obj-y := head.o als.o startup.o physmem_info.o ipl_parm.o ipl_report.o vmem.o obj-y += string.o ebcdic.o sclp_early_core.o mem.o ipl_vmparm.o cmdline.o -obj-y += version.o pgm_check_info.o ctype.o ipl_data.o machine_kexec_reloc.o +obj-y += version.o pgm_check_info.o ctype.o ipl_data.o +obj-y += $(if $(CONFIG_PIE_BUILD),machine_kexec_reloc.o,relocs.o) obj-$(findstring y, $(CONFIG_PROTECTED_VIRTUALIZATION_GUEST) $(CONFIG_PGSTE)) += uv.o obj-$(CONFIG_RANDOMIZE_BASE) += kaslr.o obj-y += $(if $(CONFIG_KERNEL_UNCOMPRESSED),,decompressor.o) info.o @@ -48,6 +49,9 @@ targets := bzImage section_cmp.boot.data section_cmp.boot.preserved.data $(obj-y targets += vmlinux.lds vmlinux vmlinux.bin vmlinux.bin.gz vmlinux.bin.bz2 targets += vmlinux.bin.xz vmlinux.bin.lzma vmlinux.bin.lzo vmlinux.bin.lz4 targets += vmlinux.bin.zst info.bin syms.bin vmlinux.syms $(obj-all) +ifndef CONFIG_PIE_BUILD +targets += relocs.S +endif OBJECTS := $(addprefix $(obj)/,$(obj-y)) OBJECTS_ALL := $(addprefix $(obj)/,$(obj-all)) @@ -56,9 +60,9 @@ clean-files += vmlinux.map quiet_cmd_section_cmp = SECTCMP $* define cmd_section_cmp - s1=`$(OBJDUMP) -t -j "$*" "$<" | sort | \ + s1=`$(OBJDUMP) -t "$<" | grep "\s$*\s\+" | sort | \ sed -n "/0000000000000000/! s/.*\s$*\s\+//p" | sha256sum`; \ - s2=`$(OBJDUMP) -t -j "$*" "$(word 2,$^)" | sort | \ + s2=`$(OBJDUMP) -t "$(word 2,$^)" | grep "\s$*\s\+" | sort | \ sed -n "/0000000000000000/! s/.*\s$*\s\+//p" | sha256sum`; \ if [ "$$s1" != "$$s2" ]; then \ echo "error: section $* differs between $< and $(word 2,$^)" >&2; \ @@ -73,11 +77,12 @@ $(obj)/bzImage: $(obj)/vmlinux $(obj)/section_cmp.boot.data $(obj)/section_cmp.b $(obj)/section_cmp%: vmlinux $(obj)/vmlinux FORCE $(call if_changed,section_cmp) -LDFLAGS_vmlinux := --oformat $(LD_BFD) -e startup $(if $(CONFIG_VMLINUX_MAP),-Map=$(obj)/vmlinux.map) --build-id=sha1 -T +LDFLAGS_vmlinux-$(CONFIG_LD_ORPHAN_WARN) := --orphan-handling=$(CONFIG_LD_ORPHAN_WARN_LEVEL) +LDFLAGS_vmlinux := $(LDFLAGS_vmlinux-y) --oformat $(LD_BFD) -e startup $(if $(CONFIG_VMLINUX_MAP),-Map=$(obj)/vmlinux.map) --build-id=sha1 -T $(obj)/vmlinux: $(obj)/vmlinux.lds $(OBJECTS_ALL) FORCE $(call if_changed,ld) -LDFLAGS_vmlinux.syms := --oformat $(LD_BFD) -e startup -T +LDFLAGS_vmlinux.syms := $(LDFLAGS_vmlinux-y) --oformat $(LD_BFD) -e startup -T $(obj)/vmlinux.syms: $(obj)/vmlinux.lds $(OBJECTS) FORCE $(call if_changed,ld) @@ -93,7 +98,7 @@ OBJCOPYFLAGS_syms.o := -I binary -O elf64-s390 -B s390:64-bit --rename-section . $(obj)/syms.o: $(obj)/syms.bin FORCE $(call if_changed,objcopy) -OBJCOPYFLAGS_info.bin := -O binary --only-section=.vmlinux.info --set-section-flags .vmlinux.info=load +OBJCOPYFLAGS_info.bin := -O binary --only-section=.vmlinux.info --set-section-flags .vmlinux.info=alloc,load $(obj)/info.bin: vmlinux FORCE $(call if_changed,objcopy) @@ -105,6 +110,14 @@ OBJCOPYFLAGS_vmlinux.bin := -O binary --remove-section=.comment --remove-section $(obj)/vmlinux.bin: vmlinux FORCE $(call if_changed,objcopy) +ifndef CONFIG_PIE_BUILD +CMD_RELOCS=arch/s390/tools/relocs +quiet_cmd_relocs = RELOCS $@ + cmd_relocs = $(CMD_RELOCS) $< > $@ +$(obj)/relocs.S: vmlinux FORCE + $(call if_changed,relocs) +endif + suffix-$(CONFIG_KERNEL_GZIP) := .gz suffix-$(CONFIG_KERNEL_BZIP2) := .bz2 suffix-$(CONFIG_KERNEL_LZ4) := .lz4 diff --git a/arch/s390/boot/boot.h b/arch/s390/boot/boot.h index 222c6886acf6..567d60f78bbc 100644 --- a/arch/s390/boot/boot.h +++ b/arch/s390/boot/boot.h @@ -25,9 +25,14 @@ struct vmlinux_info { unsigned long bootdata_size; unsigned long bootdata_preserved_off; unsigned long bootdata_preserved_size; +#ifdef CONFIG_PIE_BUILD unsigned long dynsym_start; unsigned long rela_dyn_start; unsigned long rela_dyn_end; +#else + unsigned long got_start; + unsigned long got_end; +#endif unsigned long amode31_size; unsigned long init_mm_off; unsigned long swapper_pg_dir_off; @@ -83,6 +88,7 @@ extern unsigned long vmalloc_size; extern int vmalloc_size_set; extern char __boot_data_start[], __boot_data_end[]; extern char __boot_data_preserved_start[], __boot_data_preserved_end[]; +extern char __vmlinux_relocs_64_start[], __vmlinux_relocs_64_end[]; extern char _decompressor_syms_start[], _decompressor_syms_end[]; extern char _stack_start[], _stack_end[]; extern char _end[], _decompressor_end[]; diff --git a/arch/s390/boot/startup.c b/arch/s390/boot/startup.c index 9cc76e631759..6cf89314209a 100644 --- a/arch/s390/boot/startup.c +++ b/arch/s390/boot/startup.c @@ -141,7 +141,8 @@ static void copy_bootdata(void) memcpy((void *)vmlinux.bootdata_preserved_off, __boot_data_preserved_start, vmlinux.bootdata_preserved_size); } -static void handle_relocs(unsigned long offset) +#ifdef CONFIG_PIE_BUILD +static void kaslr_adjust_relocs(unsigned long min_addr, unsigned long max_addr, unsigned long offset) { Elf64_Rela *rela_start, *rela_end, *rela; int r_type, r_sym, rc; @@ -172,6 +173,54 @@ static void handle_relocs(unsigned long offset) } } +static void kaslr_adjust_got(unsigned long offset) {} +static void rescue_relocs(void) {} +static void free_relocs(void) {} +#else +static int *vmlinux_relocs_64_start; +static int *vmlinux_relocs_64_end; + +static void rescue_relocs(void) +{ + unsigned long size = __vmlinux_relocs_64_end - __vmlinux_relocs_64_start; + + vmlinux_relocs_64_start = (void *)physmem_alloc_top_down(RR_RELOC, size, 0); + vmlinux_relocs_64_end = (void *)vmlinux_relocs_64_start + size; + memmove(vmlinux_relocs_64_start, __vmlinux_relocs_64_start, size); +} + +static void free_relocs(void) +{ + physmem_free(RR_RELOC); +} + +static void kaslr_adjust_relocs(unsigned long min_addr, unsigned long max_addr, unsigned long offset) +{ + int *reloc; + long loc; + + /* Adjust R_390_64 relocations */ + for (reloc = vmlinux_relocs_64_start; reloc < vmlinux_relocs_64_end; reloc++) { + loc = (long)*reloc + offset; + if (loc < min_addr || loc > max_addr) + error("64-bit relocation outside of kernel!\n"); + *(u64 *)loc += offset; + } +} + +static void kaslr_adjust_got(unsigned long offset) +{ + u64 *entry; + + /* + * Even without -fPIE, Clang still uses a global offset table for some + * reason. Adjust the GOT entries. + */ + for (entry = (u64 *)vmlinux.got_start; entry < (u64 *)vmlinux.got_end; entry++) + *entry += offset; +} +#endif + /* * Merge information from several sources into a single ident_map_size value. * "ident_map_size" represents the upper limit of physical memory we may ever @@ -299,14 +348,19 @@ static void setup_vmalloc_size(void) vmalloc_size = max(size, vmalloc_size); } -static void offset_vmlinux_info(unsigned long offset) +static void kaslr_adjust_vmlinux_info(unsigned long offset) { *(unsigned long *)(&vmlinux.entry) += offset; vmlinux.bootdata_off += offset; vmlinux.bootdata_preserved_off += offset; +#ifdef CONFIG_PIE_BUILD vmlinux.rela_dyn_start += offset; vmlinux.rela_dyn_end += offset; vmlinux.dynsym_start += offset; +#else + vmlinux.got_start += offset; + vmlinux.got_end += offset; +#endif vmlinux.init_mm_off += offset; vmlinux.swapper_pg_dir_off += offset; vmlinux.invalid_pg_dir_off += offset; @@ -361,6 +415,7 @@ void startup_kernel(void) detect_physmem_online_ranges(max_physmem_end); save_ipl_cert_comp_list(); rescue_initrd(safe_addr, ident_map_size); + rescue_relocs(); if (kaslr_enabled()) { vmlinux_lma = randomize_within_range(vmlinux.image_size + vmlinux.bss_size, @@ -368,7 +423,7 @@ void startup_kernel(void) ident_map_size); if (vmlinux_lma) { __kaslr_offset = vmlinux_lma - vmlinux.default_lma; - offset_vmlinux_info(__kaslr_offset); + kaslr_adjust_vmlinux_info(__kaslr_offset); } } vmlinux_lma = vmlinux_lma ?: vmlinux.default_lma; @@ -393,18 +448,20 @@ void startup_kernel(void) /* * The order of the following operations is important: * - * - handle_relocs() must follow clear_bss_section() to establish static - * memory references to data in .bss to be used by setup_vmem() + * - kaslr_adjust_relocs() must follow clear_bss_section() to establish + * static memory references to data in .bss to be used by setup_vmem() * (i.e init_mm.pgd) * - * - setup_vmem() must follow handle_relocs() to be able using + * - setup_vmem() must follow kaslr_adjust_relocs() to be able using * static memory references to data in .bss (i.e init_mm.pgd) * - * - copy_bootdata() must follow setup_vmem() to propagate changes to - * bootdata made by setup_vmem() + * - copy_bootdata() must follow setup_vmem() to propagate changes + * to bootdata made by setup_vmem() */ clear_bss_section(vmlinux_lma); - handle_relocs(__kaslr_offset); + kaslr_adjust_relocs(vmlinux_lma, vmlinux_lma + vmlinux.image_size, __kaslr_offset); + kaslr_adjust_got(__kaslr_offset); + free_relocs(); setup_vmem(asce_limit); copy_bootdata(); diff --git a/arch/s390/boot/vmlinux.lds.S b/arch/s390/boot/vmlinux.lds.S index 389df0e0d9e5..3d7ea585ab99 100644 --- a/arch/s390/boot/vmlinux.lds.S +++ b/arch/s390/boot/vmlinux.lds.S @@ -31,6 +31,7 @@ SECTIONS _text = .; /* Text */ *(.text) *(.text.*) + INIT_TEXT _etext = . ; } .rodata : { @@ -39,6 +40,9 @@ SECTIONS *(.rodata.*) _erodata = . ; } + .got : { + *(.got) + } NOTES .data : { _data = . ; @@ -106,6 +110,24 @@ SECTIONS _compressed_end = .; } +#ifndef CONFIG_PIE_BUILD + /* + * When the kernel is built with CONFIG_KERNEL_UNCOMPRESSED, the entire + * uncompressed vmlinux.bin is positioned in the bzImage decompressor + * image at the default kernel LMA of 0x100000, enabling it to be + * executed in-place. However, the size of .vmlinux.relocs could be + * large enough to cause an overlap with the uncompressed kernel at the + * address 0x100000. To address this issue, .vmlinux.relocs is + * positioned after the .rodata.compressed. + */ + . = ALIGN(4); + .vmlinux.relocs : { + __vmlinux_relocs_64_start = .; + *(.vmlinux.relocs_64) + __vmlinux_relocs_64_end = .; + } +#endif + #define SB_TRAILER_SIZE 32 /* Trailer needed for Secure Boot */ . += SB_TRAILER_SIZE; /* make sure .sb.trailer does not overwrite the previous section */ @@ -118,8 +140,34 @@ SECTIONS } _end = .; + DWARF_DEBUG + ELF_DETAILS + + /* + * Make sure that the .got.plt is either completely empty or it + * contains only the three reserved double words. + */ + .got.plt : { + *(.got.plt) + } + ASSERT(SIZEOF(.got.plt) == 0 || SIZEOF(.got.plt) == 0x18, "Unexpected GOT/PLT entries detected!") + + /* + * Sections that should stay zero sized, which is safer to + * explicitly check instead of blindly discarding. + */ + .plt : { + *(.plt) *(.plt.*) *(.iplt) *(.igot .igot.plt) + } + ASSERT(SIZEOF(.plt) == 0, "Unexpected run-time procedure linkages detected!") + .rela.dyn : { + *(.rela.*) *(.rela_*) + } + ASSERT(SIZEOF(.rela.dyn) == 0, "Unexpected run-time relocations (.rela) detected!") + /* Sections to be discarded */ /DISCARD/ : { + COMMON_DISCARDS *(.eh_frame) *(__ex_table) *(*__ksymtab*) diff --git a/arch/s390/configs/debug_defconfig b/arch/s390/configs/debug_defconfig index 06756bad5e30..4032e6e136ac 100644 --- a/arch/s390/configs/debug_defconfig +++ b/arch/s390/configs/debug_defconfig @@ -824,6 +824,8 @@ CONFIG_TEST_LOCKUP=m CONFIG_DEBUG_PREEMPT=y CONFIG_PROVE_LOCKING=y CONFIG_LOCK_STAT=y +CONFIG_LOCKDEP_BITS=16 +CONFIG_LOCKDEP_CHAINS_BITS=17 CONFIG_DEBUG_ATOMIC_SLEEP=y CONFIG_DEBUG_LOCKING_API_SELFTESTS=y CONFIG_DEBUG_IRQFLAGS=y diff --git a/arch/s390/crypto/chacha-glue.c b/arch/s390/crypto/chacha-glue.c index ed9959e6f714..f8b0c52e77a4 100644 --- a/arch/s390/crypto/chacha-glue.c +++ b/arch/s390/crypto/chacha-glue.c @@ -15,14 +15,14 @@ #include <linux/kernel.h> #include <linux/module.h> #include <linux/sizes.h> -#include <asm/fpu/api.h> +#include <asm/fpu.h> #include "chacha-s390.h" static void chacha20_crypt_s390(u32 *state, u8 *dst, const u8 *src, unsigned int nbytes, const u32 *key, u32 *counter) { - struct kernel_fpu vxstate; + DECLARE_KERNEL_FPU_ONSTACK32(vxstate); kernel_fpu_begin(&vxstate, KERNEL_VXR); chacha20_vx(dst, src, nbytes, key, counter); diff --git a/arch/s390/crypto/chacha-s390.S b/arch/s390/crypto/chacha-s390.S index 37cb63f25b17..63f3102678c0 100644 --- a/arch/s390/crypto/chacha-s390.S +++ b/arch/s390/crypto/chacha-s390.S @@ -8,7 +8,7 @@ #include <linux/linkage.h> #include <asm/nospec-insn.h> -#include <asm/vx-insn.h> +#include <asm/fpu-insn.h> #define SP %r15 #define FRAME (16 * 8 + 4 * 8) diff --git a/arch/s390/crypto/crc32-vx.c b/arch/s390/crypto/crc32-vx.c index 017143e9cef7..74f17c905d12 100644 --- a/arch/s390/crypto/crc32-vx.c +++ b/arch/s390/crypto/crc32-vx.c @@ -13,8 +13,8 @@ #include <linux/cpufeature.h> #include <linux/crc32.h> #include <crypto/internal/hash.h> -#include <asm/fpu/api.h> - +#include <asm/fpu.h> +#include "crc32-vx.h" #define CRC32_BLOCK_SIZE 1 #define CRC32_DIGEST_SIZE 4 @@ -31,11 +31,6 @@ struct crc_desc_ctx { u32 crc; }; -/* Prototypes for functions in assembly files */ -u32 crc32_le_vgfm_16(u32 crc, unsigned char const *buf, size_t size); -u32 crc32_be_vgfm_16(u32 crc, unsigned char const *buf, size_t size); -u32 crc32c_le_vgfm_16(u32 crc, unsigned char const *buf, size_t size); - /* * DEFINE_CRC32_VX() - Define a CRC-32 function using the vector extension * @@ -49,8 +44,8 @@ u32 crc32c_le_vgfm_16(u32 crc, unsigned char const *buf, size_t size); static u32 __pure ___fname(u32 crc, \ unsigned char const *data, size_t datalen) \ { \ - struct kernel_fpu vxstate; \ unsigned long prealign, aligned, remaining; \ + DECLARE_KERNEL_FPU_ONSTACK16(vxstate); \ \ if (datalen < VX_MIN_LEN + VX_ALIGN_MASK) \ return ___crc32_sw(crc, data, datalen); \ diff --git a/arch/s390/crypto/crc32-vx.h b/arch/s390/crypto/crc32-vx.h new file mode 100644 index 000000000000..652c96e1a822 --- /dev/null +++ b/arch/s390/crypto/crc32-vx.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _CRC32_VX_S390_H +#define _CRC32_VX_S390_H + +#include <linux/types.h> + +u32 crc32_be_vgfm_16(u32 crc, unsigned char const *buf, size_t size); +u32 crc32_le_vgfm_16(u32 crc, unsigned char const *buf, size_t size); +u32 crc32c_le_vgfm_16(u32 crc, unsigned char const *buf, size_t size); + +#endif /* _CRC32_VX_S390_H */ diff --git a/arch/s390/crypto/crc32be-vx.S b/arch/s390/crypto/crc32be-vx.c index 34ee47926891..fed7c9c70d05 100644 --- a/arch/s390/crypto/crc32be-vx.S +++ b/arch/s390/crypto/crc32be-vx.c @@ -12,20 +12,17 @@ * Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com> */ -#include <linux/linkage.h> -#include <asm/nospec-insn.h> -#include <asm/vx-insn.h> +#include <linux/types.h> +#include <asm/fpu.h> +#include "crc32-vx.h" /* Vector register range containing CRC-32 constants */ -#define CONST_R1R2 %v9 -#define CONST_R3R4 %v10 -#define CONST_R5 %v11 -#define CONST_R6 %v12 -#define CONST_RU_POLY %v13 -#define CONST_CRC_POLY %v14 - - .data - .balign 8 +#define CONST_R1R2 9 +#define CONST_R3R4 10 +#define CONST_R5 11 +#define CONST_R6 12 +#define CONST_RU_POLY 13 +#define CONST_CRC_POLY 14 /* * The CRC-32 constant block contains reduction constants to fold and @@ -58,105 +55,74 @@ * P'(x) = 0xEDB88320 */ -SYM_DATA_START_LOCAL(constants_CRC_32_BE) - .quad 0x08833794c, 0x0e6228b11 # R1, R2 - .quad 0x0c5b9cd4c, 0x0e8a45605 # R3, R4 - .quad 0x0f200aa66, 1 << 32 # R5, x32 - .quad 0x0490d678d, 1 # R6, 1 - .quad 0x104d101df, 0 # u - .quad 0x104C11DB7, 0 # P(x) -SYM_DATA_END(constants_CRC_32_BE) - - .previous - - GEN_BR_THUNK %r14 - - .text -/* - * The CRC-32 function(s) use these calling conventions: - * - * Parameters: - * - * %r2: Initial CRC value, typically ~0; and final CRC (return) value. - * %r3: Input buffer pointer, performance might be improved if the - * buffer is on a doubleword boundary. - * %r4: Length of the buffer, must be 64 bytes or greater. +static unsigned long constants_CRC_32_BE[] = { + 0x08833794c, 0x0e6228b11, /* R1, R2 */ + 0x0c5b9cd4c, 0x0e8a45605, /* R3, R4 */ + 0x0f200aa66, 1UL << 32, /* R5, x32 */ + 0x0490d678d, 1, /* R6, 1 */ + 0x104d101df, 0, /* u */ + 0x104C11DB7, 0, /* P(x) */ +}; + +/** + * crc32_be_vgfm_16 - Compute CRC-32 (BE variant) with vector registers + * @crc: Initial CRC value, typically ~0. + * @buf: Input buffer pointer, performance might be improved if the + * buffer is on a doubleword boundary. + * @size: Size of the buffer, must be 64 bytes or greater. * * Register usage: - * - * %r5: CRC-32 constant pool base pointer. * V0: Initial CRC value and intermediate constants and results. * V1..V4: Data for CRC computation. * V5..V8: Next data chunks that are fetched from the input buffer. - * * V9..V14: CRC-32 constants. */ -SYM_FUNC_START(crc32_be_vgfm_16) +u32 crc32_be_vgfm_16(u32 crc, unsigned char const *buf, size_t size) +{ /* Load CRC-32 constants */ - larl %r5,constants_CRC_32_BE - VLM CONST_R1R2,CONST_CRC_POLY,0,%r5 + fpu_vlm(CONST_R1R2, CONST_CRC_POLY, &constants_CRC_32_BE); + fpu_vzero(0); /* Load the initial CRC value into the leftmost word of V0. */ - VZERO %v0 - VLVGF %v0,%r2,0 + fpu_vlvgf(0, crc, 0); /* Load a 64-byte data chunk and XOR with CRC */ - VLM %v1,%v4,0,%r3 /* 64-bytes into V1..V4 */ - VX %v1,%v0,%v1 /* V1 ^= CRC */ - aghi %r3,64 /* BUF = BUF + 64 */ - aghi %r4,-64 /* LEN = LEN - 64 */ - - /* Check remaining buffer size and jump to proper folding method */ - cghi %r4,64 - jl .Lless_than_64bytes - -.Lfold_64bytes_loop: - /* Load the next 64-byte data chunk into V5 to V8 */ - VLM %v5,%v8,0,%r3 + fpu_vlm(1, 4, buf); + fpu_vx(1, 0, 1); + buf += 64; + size -= 64; + + while (size >= 64) { + /* Load the next 64-byte data chunk into V5 to V8 */ + fpu_vlm(5, 8, buf); + + /* + * Perform a GF(2) multiplication of the doublewords in V1 with + * the reduction constants in V0. The intermediate result is + * then folded (accumulated) with the next data chunk in V5 and + * stored in V1. Repeat this step for the register contents + * in V2, V3, and V4 respectively. + */ + fpu_vgfmag(1, CONST_R1R2, 1, 5); + fpu_vgfmag(2, CONST_R1R2, 2, 6); + fpu_vgfmag(3, CONST_R1R2, 3, 7); + fpu_vgfmag(4, CONST_R1R2, 4, 8); + buf += 64; + size -= 64; + } - /* - * Perform a GF(2) multiplication of the doublewords in V1 with - * the reduction constants in V0. The intermediate result is - * then folded (accumulated) with the next data chunk in V5 and - * stored in V1. Repeat this step for the register contents - * in V2, V3, and V4 respectively. - */ - VGFMAG %v1,CONST_R1R2,%v1,%v5 - VGFMAG %v2,CONST_R1R2,%v2,%v6 - VGFMAG %v3,CONST_R1R2,%v3,%v7 - VGFMAG %v4,CONST_R1R2,%v4,%v8 - - /* Adjust buffer pointer and length for next loop */ - aghi %r3,64 /* BUF = BUF + 64 */ - aghi %r4,-64 /* LEN = LEN - 64 */ - - cghi %r4,64 - jnl .Lfold_64bytes_loop - -.Lless_than_64bytes: /* Fold V1 to V4 into a single 128-bit value in V1 */ - VGFMAG %v1,CONST_R3R4,%v1,%v2 - VGFMAG %v1,CONST_R3R4,%v1,%v3 - VGFMAG %v1,CONST_R3R4,%v1,%v4 - - /* Check whether to continue with 64-bit folding */ - cghi %r4,16 - jl .Lfinal_fold + fpu_vgfmag(1, CONST_R3R4, 1, 2); + fpu_vgfmag(1, CONST_R3R4, 1, 3); + fpu_vgfmag(1, CONST_R3R4, 1, 4); -.Lfold_16bytes_loop: + while (size >= 16) { + fpu_vl(2, buf); + fpu_vgfmag(1, CONST_R3R4, 1, 2); + buf += 16; + size -= 16; + } - VL %v2,0,,%r3 /* Load next data chunk */ - VGFMAG %v1,CONST_R3R4,%v1,%v2 /* Fold next data chunk */ - - /* Adjust buffer pointer and size for folding next data chunk */ - aghi %r3,16 - aghi %r4,-16 - - /* Process remaining data chunks */ - cghi %r4,16 - jnl .Lfold_16bytes_loop - -.Lfinal_fold: /* * The R5 constant is used to fold a 128-bit value into an 96-bit value * that is XORed with the next 96-bit input data chunk. To use a single @@ -164,7 +130,7 @@ SYM_FUNC_START(crc32_be_vgfm_16) * form an intermediate 96-bit value (with appended zeros) which is then * XORed with the intermediate reduction result. */ - VGFMG %v1,CONST_R5,%v1 + fpu_vgfmg(1, CONST_R5, 1); /* * Further reduce the remaining 96-bit value to a 64-bit value using a @@ -173,7 +139,7 @@ SYM_FUNC_START(crc32_be_vgfm_16) * doubleword with R6. The result is a 64-bit value and is subject to * the Barret reduction. */ - VGFMG %v1,CONST_R6,%v1 + fpu_vgfmg(1, CONST_R6, 1); /* * The input values to the Barret reduction are the degree-63 polynomial @@ -194,20 +160,15 @@ SYM_FUNC_START(crc32_be_vgfm_16) */ /* T1(x) = floor( R(x) / x^32 ) GF2MUL u */ - VUPLLF %v2,%v1 - VGFMG %v2,CONST_RU_POLY,%v2 + fpu_vupllf(2, 1); + fpu_vgfmg(2, CONST_RU_POLY, 2); /* * Compute the GF(2) product of the CRC polynomial in VO with T1(x) in * V2 and XOR the intermediate result, T2(x), with the value in V1. * The final result is in the rightmost word of V2. */ - VUPLLF %v2,%v2 - VGFMAG %v2,CONST_CRC_POLY,%v2,%v1 - -.Ldone: - VLGVF %r2,%v2,3 - BR_EX %r14 -SYM_FUNC_END(crc32_be_vgfm_16) - -.previous + fpu_vupllf(2, 2); + fpu_vgfmag(2, CONST_CRC_POLY, 2, 1); + return fpu_vlgvf(2, 3); +} diff --git a/arch/s390/crypto/crc32le-vx.S b/arch/s390/crypto/crc32le-vx.c index 5a819ae09a0b..2f629f394df7 100644 --- a/arch/s390/crypto/crc32le-vx.S +++ b/arch/s390/crypto/crc32le-vx.c @@ -13,20 +13,17 @@ * Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com> */ -#include <linux/linkage.h> -#include <asm/nospec-insn.h> -#include <asm/vx-insn.h> +#include <linux/types.h> +#include <asm/fpu.h> +#include "crc32-vx.h" /* Vector register range containing CRC-32 constants */ -#define CONST_PERM_LE2BE %v9 -#define CONST_R2R1 %v10 -#define CONST_R4R3 %v11 -#define CONST_R5 %v12 -#define CONST_RU_POLY %v13 -#define CONST_CRC_POLY %v14 - - .data - .balign 8 +#define CONST_PERM_LE2BE 9 +#define CONST_R2R1 10 +#define CONST_R4R3 11 +#define CONST_R5 12 +#define CONST_RU_POLY 13 +#define CONST_CRC_POLY 14 /* * The CRC-32 constant block contains reduction constants to fold and @@ -59,64 +56,43 @@ * P'(x) = 0x82F63B78 */ -SYM_DATA_START_LOCAL(constants_CRC_32_LE) - .octa 0x0F0E0D0C0B0A09080706050403020100 # BE->LE mask - .quad 0x1c6e41596, 0x154442bd4 # R2, R1 - .quad 0x0ccaa009e, 0x1751997d0 # R4, R3 - .octa 0x163cd6124 # R5 - .octa 0x1F7011641 # u' - .octa 0x1DB710641 # P'(x) << 1 -SYM_DATA_END(constants_CRC_32_LE) - -SYM_DATA_START_LOCAL(constants_CRC_32C_LE) - .octa 0x0F0E0D0C0B0A09080706050403020100 # BE->LE mask - .quad 0x09e4addf8, 0x740eef02 # R2, R1 - .quad 0x14cd00bd6, 0xf20c0dfe # R4, R3 - .octa 0x0dd45aab8 # R5 - .octa 0x0dea713f1 # u' - .octa 0x105ec76f0 # P'(x) << 1 -SYM_DATA_END(constants_CRC_32C_LE) - - .previous - - GEN_BR_THUNK %r14 - - .text - -/* - * The CRC-32 functions use these calling conventions: - * - * Parameters: - * - * %r2: Initial CRC value, typically ~0; and final CRC (return) value. - * %r3: Input buffer pointer, performance might be improved if the - * buffer is on a doubleword boundary. - * %r4: Length of the buffer, must be 64 bytes or greater. +static unsigned long constants_CRC_32_LE[] = { + 0x0f0e0d0c0b0a0908, 0x0706050403020100, /* BE->LE mask */ + 0x1c6e41596, 0x154442bd4, /* R2, R1 */ + 0x0ccaa009e, 0x1751997d0, /* R4, R3 */ + 0x0, 0x163cd6124, /* R5 */ + 0x0, 0x1f7011641, /* u' */ + 0x0, 0x1db710641 /* P'(x) << 1 */ +}; + +static unsigned long constants_CRC_32C_LE[] = { + 0x0f0e0d0c0b0a0908, 0x0706050403020100, /* BE->LE mask */ + 0x09e4addf8, 0x740eef02, /* R2, R1 */ + 0x14cd00bd6, 0xf20c0dfe, /* R4, R3 */ + 0x0, 0x0dd45aab8, /* R5 */ + 0x0, 0x0dea713f1, /* u' */ + 0x0, 0x105ec76f0 /* P'(x) << 1 */ +}; + +/** + * crc32_le_vgfm_generic - Compute CRC-32 (LE variant) with vector registers + * @crc: Initial CRC value, typically ~0. + * @buf: Input buffer pointer, performance might be improved if the + * buffer is on a doubleword boundary. + * @size: Size of the buffer, must be 64 bytes or greater. + * @constants: CRC-32 constant pool base pointer. * * Register usage: - * - * %r5: CRC-32 constant pool base pointer. - * V0: Initial CRC value and intermediate constants and results. - * V1..V4: Data for CRC computation. - * V5..V8: Next data chunks that are fetched from the input buffer. - * V9: Constant for BE->LE conversion and shift operations - * + * V0: Initial CRC value and intermediate constants and results. + * V1..V4: Data for CRC computation. + * V5..V8: Next data chunks that are fetched from the input buffer. + * V9: Constant for BE->LE conversion and shift operations * V10..V14: CRC-32 constants. */ - -SYM_FUNC_START(crc32_le_vgfm_16) - larl %r5,constants_CRC_32_LE - j crc32_le_vgfm_generic -SYM_FUNC_END(crc32_le_vgfm_16) - -SYM_FUNC_START(crc32c_le_vgfm_16) - larl %r5,constants_CRC_32C_LE - j crc32_le_vgfm_generic -SYM_FUNC_END(crc32c_le_vgfm_16) - -SYM_FUNC_START(crc32_le_vgfm_generic) +static u32 crc32_le_vgfm_generic(u32 crc, unsigned char const *buf, size_t size, unsigned long *constants) +{ /* Load CRC-32 constants */ - VLM CONST_PERM_LE2BE,CONST_CRC_POLY,0,%r5 + fpu_vlm(CONST_PERM_LE2BE, CONST_CRC_POLY, constants); /* * Load the initial CRC value. @@ -125,90 +101,73 @@ SYM_FUNC_START(crc32_le_vgfm_generic) * vector register and is later XORed with the LSB portion * of the loaded input data. */ - VZERO %v0 /* Clear V0 */ - VLVGF %v0,%r2,3 /* Load CRC into rightmost word */ + fpu_vzero(0); /* Clear V0 */ + fpu_vlvgf(0, crc, 3); /* Load CRC into rightmost word */ /* Load a 64-byte data chunk and XOR with CRC */ - VLM %v1,%v4,0,%r3 /* 64-bytes into V1..V4 */ - VPERM %v1,%v1,%v1,CONST_PERM_LE2BE - VPERM %v2,%v2,%v2,CONST_PERM_LE2BE - VPERM %v3,%v3,%v3,CONST_PERM_LE2BE - VPERM %v4,%v4,%v4,CONST_PERM_LE2BE + fpu_vlm(1, 4, buf); + fpu_vperm(1, 1, 1, CONST_PERM_LE2BE); + fpu_vperm(2, 2, 2, CONST_PERM_LE2BE); + fpu_vperm(3, 3, 3, CONST_PERM_LE2BE); + fpu_vperm(4, 4, 4, CONST_PERM_LE2BE); + + fpu_vx(1, 0, 1); /* V1 ^= CRC */ + buf += 64; + size -= 64; + + while (size >= 64) { + fpu_vlm(5, 8, buf); + fpu_vperm(5, 5, 5, CONST_PERM_LE2BE); + fpu_vperm(6, 6, 6, CONST_PERM_LE2BE); + fpu_vperm(7, 7, 7, CONST_PERM_LE2BE); + fpu_vperm(8, 8, 8, CONST_PERM_LE2BE); + /* + * Perform a GF(2) multiplication of the doublewords in V1 with + * the R1 and R2 reduction constants in V0. The intermediate + * result is then folded (accumulated) with the next data chunk + * in V5 and stored in V1. Repeat this step for the register + * contents in V2, V3, and V4 respectively. + */ + fpu_vgfmag(1, CONST_R2R1, 1, 5); + fpu_vgfmag(2, CONST_R2R1, 2, 6); + fpu_vgfmag(3, CONST_R2R1, 3, 7); + fpu_vgfmag(4, CONST_R2R1, 4, 8); + buf += 64; + size -= 64; + } - VX %v1,%v0,%v1 /* V1 ^= CRC */ - aghi %r3,64 /* BUF = BUF + 64 */ - aghi %r4,-64 /* LEN = LEN - 64 */ - - cghi %r4,64 - jl .Lless_than_64bytes - -.Lfold_64bytes_loop: - /* Load the next 64-byte data chunk into V5 to V8 */ - VLM %v5,%v8,0,%r3 - VPERM %v5,%v5,%v5,CONST_PERM_LE2BE - VPERM %v6,%v6,%v6,CONST_PERM_LE2BE - VPERM %v7,%v7,%v7,CONST_PERM_LE2BE - VPERM %v8,%v8,%v8,CONST_PERM_LE2BE - - /* - * Perform a GF(2) multiplication of the doublewords in V1 with - * the R1 and R2 reduction constants in V0. The intermediate result - * is then folded (accumulated) with the next data chunk in V5 and - * stored in V1. Repeat this step for the register contents - * in V2, V3, and V4 respectively. - */ - VGFMAG %v1,CONST_R2R1,%v1,%v5 - VGFMAG %v2,CONST_R2R1,%v2,%v6 - VGFMAG %v3,CONST_R2R1,%v3,%v7 - VGFMAG %v4,CONST_R2R1,%v4,%v8 - - aghi %r3,64 /* BUF = BUF + 64 */ - aghi %r4,-64 /* LEN = LEN - 64 */ - - cghi %r4,64 - jnl .Lfold_64bytes_loop - -.Lless_than_64bytes: /* * Fold V1 to V4 into a single 128-bit value in V1. Multiply V1 with R3 * and R4 and accumulating the next 128-bit chunk until a single 128-bit * value remains. */ - VGFMAG %v1,CONST_R4R3,%v1,%v2 - VGFMAG %v1,CONST_R4R3,%v1,%v3 - VGFMAG %v1,CONST_R4R3,%v1,%v4 - - cghi %r4,16 - jl .Lfinal_fold - -.Lfold_16bytes_loop: - - VL %v2,0,,%r3 /* Load next data chunk */ - VPERM %v2,%v2,%v2,CONST_PERM_LE2BE - VGFMAG %v1,CONST_R4R3,%v1,%v2 /* Fold next data chunk */ + fpu_vgfmag(1, CONST_R4R3, 1, 2); + fpu_vgfmag(1, CONST_R4R3, 1, 3); + fpu_vgfmag(1, CONST_R4R3, 1, 4); + + while (size >= 16) { + fpu_vl(2, buf); + fpu_vperm(2, 2, 2, CONST_PERM_LE2BE); + fpu_vgfmag(1, CONST_R4R3, 1, 2); + buf += 16; + size -= 16; + } - aghi %r3,16 - aghi %r4,-16 - - cghi %r4,16 - jnl .Lfold_16bytes_loop - -.Lfinal_fold: /* * Set up a vector register for byte shifts. The shift value must * be loaded in bits 1-4 in byte element 7 of a vector register. * Shift by 8 bytes: 0x40 * Shift by 4 bytes: 0x20 */ - VLEIB %v9,0x40,7 + fpu_vleib(9, 0x40, 7); /* * Prepare V0 for the next GF(2) multiplication: shift V0 by 8 bytes * to move R4 into the rightmost doubleword and set the leftmost * doubleword to 0x1. */ - VSRLB %v0,CONST_R4R3,%v9 - VLEIG %v0,1,0 + fpu_vsrlb(0, CONST_R4R3, 9); + fpu_vleig(0, 1, 0); /* * Compute GF(2) product of V1 and V0. The rightmost doubleword @@ -216,7 +175,7 @@ SYM_FUNC_START(crc32_le_vgfm_generic) * multiplied by 0x1 and is then XORed with rightmost product. * Implicitly, the intermediate leftmost product becomes padded */ - VGFMG %v1,%v0,%v1 + fpu_vgfmg(1, 0, 1); /* * Now do the final 32-bit fold by multiplying the rightmost word @@ -231,10 +190,10 @@ SYM_FUNC_START(crc32_le_vgfm_generic) * rightmost doubleword and the leftmost doubleword is zero to ignore * the leftmost product of V1. */ - VLEIB %v9,0x20,7 /* Shift by words */ - VSRLB %v2,%v1,%v9 /* Store remaining bits in V2 */ - VUPLLF %v1,%v1 /* Split rightmost doubleword */ - VGFMAG %v1,CONST_R5,%v1,%v2 /* V1 = (V1 * R5) XOR V2 */ + fpu_vleib(9, 0x20, 7); /* Shift by words */ + fpu_vsrlb(2, 1, 9); /* Store remaining bits in V2 */ + fpu_vupllf(1, 1); /* Split rightmost doubleword */ + fpu_vgfmag(1, CONST_R5, 1, 2); /* V1 = (V1 * R5) XOR V2 */ /* * Apply a Barret reduction to compute the final 32-bit CRC value. @@ -256,20 +215,26 @@ SYM_FUNC_START(crc32_le_vgfm_generic) */ /* T1(x) = floor( R(x) / x^32 ) GF2MUL u */ - VUPLLF %v2,%v1 - VGFMG %v2,CONST_RU_POLY,%v2 + fpu_vupllf(2, 1); + fpu_vgfmg(2, CONST_RU_POLY, 2); /* * Compute the GF(2) product of the CRC polynomial with T1(x) in * V2 and XOR the intermediate result, T2(x), with the value in V1. * The final result is stored in word element 2 of V2. */ - VUPLLF %v2,%v2 - VGFMAG %v2,CONST_CRC_POLY,%v2,%v1 + fpu_vupllf(2, 2); + fpu_vgfmag(2, CONST_CRC_POLY, 2, 1); + + return fpu_vlgvf(2, 2); +} -.Ldone: - VLGVF %r2,%v2,2 - BR_EX %r14 -SYM_FUNC_END(crc32_le_vgfm_generic) +u32 crc32_le_vgfm_16(u32 crc, unsigned char const *buf, size_t size) +{ + return crc32_le_vgfm_generic(crc, buf, size, &constants_CRC_32_LE[0]); +} -.previous +u32 crc32c_le_vgfm_16(u32 crc, unsigned char const *buf, size_t size) +{ + return crc32_le_vgfm_generic(crc, buf, size, &constants_CRC_32C_LE[0]); +} diff --git a/arch/s390/crypto/paes_s390.c b/arch/s390/crypto/paes_s390.c index 55ee5567a5ea..99f7e1f2b70a 100644 --- a/arch/s390/crypto/paes_s390.c +++ b/arch/s390/crypto/paes_s390.c @@ -125,20 +125,8 @@ struct s390_pxts_ctx { static inline int __paes_keyblob2pkey(struct key_blob *kb, struct pkey_protkey *pk) { - int i, ret; - - /* try three times in case of failure */ - for (i = 0; i < 3; i++) { - if (i > 0 && ret == -EAGAIN && in_task()) - if (msleep_interruptible(1000)) - return -EINTR; - ret = pkey_keyblob2pkey(kb->key, kb->keylen, - pk->protkey, &pk->len, &pk->type); - if (ret == 0) - break; - } - - return ret; + return pkey_keyblob2pkey(kb->key, kb->keylen, + pk->protkey, &pk->len, &pk->type); } static inline int __paes_convert_key(struct s390_paes_ctx *ctx) diff --git a/arch/s390/hypfs/hypfs_diag0c.c b/arch/s390/hypfs/hypfs_diag0c.c index 9a2786079e3a..4131f0daa5ea 100644 --- a/arch/s390/hypfs/hypfs_diag0c.c +++ b/arch/s390/hypfs/hypfs_diag0c.c @@ -20,8 +20,7 @@ */ static void diag0c_fn(void *data) { - diag_stat_inc(DIAG_STAT_X00C); - diag_amode31_ops.diag0c(((void **)data)[smp_processor_id()]); + diag0c(((void **)data)[smp_processor_id()]); } /* diff --git a/arch/s390/hypfs/hypfs_sprp.c b/arch/s390/hypfs/hypfs_sprp.c index f5f7e78ddc0c..9fc3f0dae8f0 100644 --- a/arch/s390/hypfs/hypfs_sprp.c +++ b/arch/s390/hypfs/hypfs_sprp.c @@ -25,7 +25,7 @@ static inline unsigned long __hypfs_sprp_diag304(void *data, unsigned long cmd) { - union register_pair r1 = { .even = (unsigned long)data, }; + union register_pair r1 = { .even = virt_to_phys(data), }; asm volatile("diag %[r1],%[r3],0x304\n" : [r1] "+&d" (r1.pair) @@ -74,7 +74,7 @@ static int __hypfs_sprp_ioctl(void __user *user_area) int rc; rc = -ENOMEM; - data = (void *) get_zeroed_page(GFP_KERNEL | GFP_DMA); + data = (void *)get_zeroed_page(GFP_KERNEL); diag304 = kzalloc(sizeof(*diag304), GFP_KERNEL); if (!data || !diag304) goto out; diff --git a/arch/s390/include/asm/access-regs.h b/arch/s390/include/asm/access-regs.h new file mode 100644 index 000000000000..1a6412d9f5ad --- /dev/null +++ b/arch/s390/include/asm/access-regs.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright IBM Corp. 1999, 2024 + */ + +#ifndef __ASM_S390_ACCESS_REGS_H +#define __ASM_S390_ACCESS_REGS_H + +#include <linux/instrumented.h> +#include <asm/sigcontext.h> + +struct access_regs { + unsigned int regs[NUM_ACRS]; +}; + +static inline void save_access_regs(unsigned int *acrs) +{ + struct access_regs *regs = (struct access_regs *)acrs; + + instrument_write(regs, sizeof(*regs)); + asm volatile("stamy 0,15,%[regs]" + : [regs] "=QS" (*regs) + : + : "memory"); +} + +static inline void restore_access_regs(unsigned int *acrs) +{ + struct access_regs *regs = (struct access_regs *)acrs; + + instrument_read(regs, sizeof(*regs)); + asm volatile("lamy 0,15,%[regs]" + : + : [regs] "QS" (*regs) + : "memory"); +} + +#endif /* __ASM_S390_ACCESS_REGS_H */ diff --git a/arch/s390/include/asm/appldata.h b/arch/s390/include/asm/appldata.h index f2240392c708..a92ebbc7aa7a 100644 --- a/arch/s390/include/asm/appldata.h +++ b/arch/s390/include/asm/appldata.h @@ -54,13 +54,13 @@ static inline int appldata_asm(struct appldata_parameter_list *parm_list, parm_list->function = fn; parm_list->parlist_length = sizeof(*parm_list); parm_list->buffer_length = length; - parm_list->product_id_addr = (unsigned long) id; + parm_list->product_id_addr = virt_to_phys(id); parm_list->buffer_addr = virt_to_phys(buffer); diag_stat_inc(DIAG_STAT_X0DC); asm volatile( " diag %1,%0,0xdc" : "=d" (ry) - : "d" (parm_list), "m" (*parm_list), "m" (*id) + : "d" (virt_to_phys(parm_list)), "m" (*parm_list), "m" (*id) : "cc"); return ry; } diff --git a/arch/s390/include/asm/asm-prototypes.h b/arch/s390/include/asm/asm-prototypes.h index a873e873e1ee..56096ae26f29 100644 --- a/arch/s390/include/asm/asm-prototypes.h +++ b/arch/s390/include/asm/asm-prototypes.h @@ -3,7 +3,7 @@ #include <linux/kvm_host.h> #include <linux/ftrace.h> -#include <asm/fpu/api.h> +#include <asm/fpu.h> #include <asm-generic/asm-prototypes.h> __int128_t __ashlti3(__int128_t a, int b); diff --git a/arch/s390/include/asm/bug.h b/arch/s390/include/asm/bug.h index aebe1e22c7be..c500d45fb465 100644 --- a/arch/s390/include/asm/bug.h +++ b/arch/s390/include/asm/bug.h @@ -14,7 +14,7 @@ ".section .rodata.str,\"aMS\",@progbits,1\n" \ "1: .asciz \""__FILE__"\"\n" \ ".previous\n" \ - ".section __bug_table,\"awM\",@progbits,%2\n" \ + ".section __bug_table,\"aw\"\n" \ "2: .long 0b-.\n" \ " .long 1b-.\n" \ " .short %0,%1\n" \ @@ -30,7 +30,7 @@ #define __EMIT_BUG(x) do { \ asm_inline volatile( \ "0: mc 0,0\n" \ - ".section __bug_table,\"awM\",@progbits,%1\n" \ + ".section __bug_table,\"aw\"\n" \ "1: .long 0b-.\n" \ " .short %0\n" \ " .org 1b+%1\n" \ diff --git a/arch/s390/include/asm/checksum.h b/arch/s390/include/asm/checksum.h index 69837eec2ff5..b89159591ca0 100644 --- a/arch/s390/include/asm/checksum.h +++ b/arch/s390/include/asm/checksum.h @@ -12,36 +12,29 @@ #ifndef _S390_CHECKSUM_H #define _S390_CHECKSUM_H -#include <linux/kasan-checks.h> +#include <linux/instrumented.h> #include <linux/in6.h> -/* - * Computes the checksum of a memory block at buff, length len, - * and adds in "sum" (32-bit). - * - * Returns a 32-bit number suitable for feeding into itself - * or csum_tcpudp_magic. - * - * This function must be called with even lengths, except - * for the last fragment, which may be odd. - * - * It's best to have buff aligned on a 32-bit boundary. - */ -static inline __wsum csum_partial(const void *buff, int len, __wsum sum) +static inline __wsum cksm(const void *buff, int len, __wsum sum) { union register_pair rp = { - .even = (unsigned long) buff, - .odd = (unsigned long) len, + .even = (unsigned long)buff, + .odd = (unsigned long)len, }; - kasan_check_read(buff, len); - asm volatile( + instrument_read(buff, len); + asm volatile("\n" "0: cksm %[sum],%[rp]\n" " jo 0b\n" : [sum] "+&d" (sum), [rp] "+&d" (rp.pair) : : "cc", "memory"); return sum; } +__wsum csum_partial(const void *buff, int len, __wsum sum); + +#define _HAVE_ARCH_CSUM_AND_COPY +__wsum csum_partial_copy_nocheck(const void *src, void *dst, int len); + /* * Fold a partial checksum without adding pseudo headers. */ diff --git a/arch/s390/include/asm/diag.h b/arch/s390/include/asm/diag.h index bed804137537..20b94220113b 100644 --- a/arch/s390/include/asm/diag.h +++ b/arch/s390/include/asm/diag.h @@ -44,6 +44,13 @@ enum diag_stat_enum { void diag_stat_inc(enum diag_stat_enum nr); void diag_stat_inc_norecursion(enum diag_stat_enum nr); +struct hypfs_diag0c_entry; + +/* + * Diagnose 0c: Pseudo Timer + */ +void diag0c(struct hypfs_diag0c_entry *data); + /* * Diagnose 10: Release page range */ @@ -331,10 +338,10 @@ struct hypfs_diag0c_entry; */ struct diag_ops { int (*diag210)(struct diag210 *addr); - int (*diag26c)(void *req, void *resp, enum diag26c_sc subcode); + int (*diag26c)(unsigned long rx, unsigned long rx1, enum diag26c_sc subcode); int (*diag14)(unsigned long rx, unsigned long ry1, unsigned long subcode); int (*diag8c)(struct diag8c *addr, struct ccw_dev_id *devno, size_t len); - void (*diag0c)(struct hypfs_diag0c_entry *entry); + void (*diag0c)(unsigned long rx); void (*diag308_reset)(void); }; @@ -342,9 +349,9 @@ extern struct diag_ops diag_amode31_ops; extern struct diag210 *__diag210_tmp_amode31; int _diag210_amode31(struct diag210 *addr); -int _diag26c_amode31(void *req, void *resp, enum diag26c_sc subcode); +int _diag26c_amode31(unsigned long rx, unsigned long rx1, enum diag26c_sc subcode); int _diag14_amode31(unsigned long rx, unsigned long ry1, unsigned long subcode); -void _diag0c_amode31(struct hypfs_diag0c_entry *entry); +void _diag0c_amode31(unsigned long rx); void _diag308_reset_amode31(void); int _diag8c_amode31(struct diag8c *addr, struct ccw_dev_id *devno, size_t len); diff --git a/arch/s390/include/asm/entry-common.h b/arch/s390/include/asm/entry-common.h index fdd319a622b0..7f5004065e8a 100644 --- a/arch/s390/include/asm/entry-common.h +++ b/arch/s390/include/asm/entry-common.h @@ -8,7 +8,7 @@ #include <linux/processor.h> #include <linux/uaccess.h> #include <asm/timex.h> -#include <asm/fpu/api.h> +#include <asm/fpu.h> #include <asm/pai.h> #define ARCH_EXIT_TO_USER_MODE_WORK (_TIF_GUARDED_STORAGE | _TIF_PER_TRAP) @@ -41,8 +41,7 @@ static __always_inline void arch_exit_to_user_mode_work(struct pt_regs *regs, static __always_inline void arch_exit_to_user_mode(void) { - if (test_cpu_flag(CIF_FPU)) - __load_fpu_regs(); + load_user_fpu_regs(); if (IS_ENABLED(CONFIG_DEBUG_ENTRY)) debug_user_asce(1); diff --git a/arch/s390/include/asm/vx-insn-asm.h b/arch/s390/include/asm/fpu-insn-asm.h index 360f8b36d962..02ccfe46050a 100644 --- a/arch/s390/include/asm/vx-insn-asm.h +++ b/arch/s390/include/asm/fpu-insn-asm.h @@ -9,11 +9,11 @@ * Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com> */ -#ifndef __ASM_S390_VX_INSN_INTERNAL_H -#define __ASM_S390_VX_INSN_INTERNAL_H +#ifndef __ASM_S390_FPU_INSN_ASM_H +#define __ASM_S390_FPU_INSN_ASM_H -#ifndef __ASM_S390_VX_INSN_H -#error only <asm/vx-insn.h> can be included directly +#ifndef __ASM_S390_FPU_INSN_H +#error only <asm/fpu-insn.h> can be included directly #endif #ifdef __ASSEMBLY__ @@ -195,10 +195,26 @@ /* RXB - Compute most significant bit used vector registers * * @rxb: Operand to store computed RXB value - * @v1: First vector register designated operand - * @v2: Second vector register designated operand - * @v3: Third vector register designated operand - * @v4: Fourth vector register designated operand + * @v1: Vector register designated operand whose MSB is stored in + * RXB bit 0 (instruction bit 36) and whose remaining bits + * are stored in instruction bits 8-11. + * @v2: Vector register designated operand whose MSB is stored in + * RXB bit 1 (instruction bit 37) and whose remaining bits + * are stored in instruction bits 12-15. + * @v3: Vector register designated operand whose MSB is stored in + * RXB bit 2 (instruction bit 38) and whose remaining bits + * are stored in instruction bits 16-19. + * @v4: Vector register designated operand whose MSB is stored in + * RXB bit 3 (instruction bit 39) and whose remaining bits + * are stored in instruction bits 32-35. + * + * Note: In most vector instruction formats [1] V1, V2, V3, and V4 directly + * correspond to @v1, @v2, @v3, and @v4. But there are exceptions, such as but + * not limited to the vector instruction formats VRR-g, VRR-h, VRS-a, VRS-d, + * and VSI. + * + * [1] IBM z/Architecture Principles of Operation, chapter "Program + * Execution, section "Instructions", subsection "Instruction Formats". */ .macro RXB rxb v1 v2=0 v3=0 v4=0 \rxb = 0 @@ -223,6 +239,9 @@ * @v2: Second vector register designated operand (for RXB) * @v3: Third vector register designated operand (for RXB) * @v4: Fourth vector register designated operand (for RXB) + * + * Note: For @v1, @v2, @v3, and @v4 also refer to the RXB macro + * description for further details. */ .macro MRXB m v1 v2=0 v3=0 v4=0 rxb = 0 @@ -238,6 +257,9 @@ * @v2: Second vector register designated operand (for RXB) * @v3: Third vector register designated operand (for RXB) * @v4: Fourth vector register designated operand (for RXB) + * + * Note: For @v1, @v2, @v3, and @v4 also refer to the RXB macro + * description for further details. */ .macro MRXBOPC m opc v1 v2=0 v3=0 v4=0 MRXB \m, \v1, \v2, \v3, \v4 @@ -350,7 +372,7 @@ VX_NUM v3, \vr .word 0xE700 | (r1 << 4) | (v3&15) .word (b2 << 12) | (\disp) - MRXBOPC \m, 0x21, v3 + MRXBOPC \m, 0x21, 0, v3 .endm .macro VLGVB gr, vr, disp, base="%r0" VLGV \gr, \vr, \disp, \base, 0 @@ -499,6 +521,25 @@ VMRL \vr1, \vr2, \vr3, 3 .endm +/* VECTOR LOAD WITH LENGTH */ +.macro VLL v, gr, disp, base + VX_NUM v1, \v + GR_NUM b2, \base + GR_NUM r3, \gr + .word 0xE700 | ((v1&15) << 4) | r3 + .word (b2 << 12) | (\disp) + MRXBOPC 0, 0x37, v1 +.endm + +/* VECTOR STORE WITH LENGTH */ +.macro VSTL v, gr, disp, base + VX_NUM v1, \v + GR_NUM b2, \base + GR_NUM r3, \gr + .word 0xE700 | ((v1&15) << 4) | r3 + .word (b2 << 12) | (\disp) + MRXBOPC 0, 0x3f, v1 +.endm /* Vector integer instructions */ @@ -512,6 +553,16 @@ MRXBOPC 0, 0x68, v1, v2, v3 .endm +/* VECTOR CHECKSUM */ +.macro VCKSM vr1, vr2, vr3 + VX_NUM v1, \vr1 + VX_NUM v2, \vr2 + VX_NUM v3, \vr3 + .word 0xE700 | ((v1&15) << 4) | (v2&15) + .word ((v3&15) << 12) + MRXBOPC 0, 0x66, v1, v2, v3 +.endm + /* VECTOR EXCLUSIVE OR */ .macro VX vr1, vr2, vr3 VX_NUM v1, \vr1 @@ -678,4 +729,4 @@ .endm #endif /* __ASSEMBLY__ */ -#endif /* __ASM_S390_VX_INSN_INTERNAL_H */ +#endif /* __ASM_S390_FPU_INSN_ASM_H */ diff --git a/arch/s390/include/asm/fpu-insn.h b/arch/s390/include/asm/fpu-insn.h new file mode 100644 index 000000000000..c1e2e521d9af --- /dev/null +++ b/arch/s390/include/asm/fpu-insn.h @@ -0,0 +1,486 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Support for Floating Point and Vector Instructions + * + */ + +#ifndef __ASM_S390_FPU_INSN_H +#define __ASM_S390_FPU_INSN_H + +#include <asm/fpu-insn-asm.h> + +#ifndef __ASSEMBLY__ + +#include <linux/instrumented.h> +#include <asm/asm-extable.h> + +asm(".include \"asm/fpu-insn-asm.h\"\n"); + +/* + * Various small helper functions, which can and should be used within + * kernel fpu code sections. Each function represents only one floating + * point or vector instruction (except for helper functions which require + * exception handling). + * + * This allows to use floating point and vector instructions like C + * functions, which has the advantage that all supporting code, like + * e.g. loops, can be written in easy to read C code. + * + * Each of the helper functions provides support for code instrumentation, + * like e.g. KASAN. Therefore instrumentation is also covered automatically + * when using these functions. + * + * In order to ensure that code generated with the helper functions stays + * within kernel fpu sections, which are guarded with kernel_fpu_begin() + * and kernel_fpu_end() calls, each function has a mandatory "memory" + * barrier. + */ + +static __always_inline void fpu_cefbr(u8 f1, s32 val) +{ + asm volatile("cefbr %[f1],%[val]\n" + : + : [f1] "I" (f1), [val] "d" (val) + : "memory"); +} + +static __always_inline unsigned long fpu_cgebr(u8 f2, u8 mode) +{ + unsigned long val; + + asm volatile("cgebr %[val],%[mode],%[f2]\n" + : [val] "=d" (val) + : [f2] "I" (f2), [mode] "I" (mode) + : "memory"); + return val; +} + +static __always_inline void fpu_debr(u8 f1, u8 f2) +{ + asm volatile("debr %[f1],%[f2]\n" + : + : [f1] "I" (f1), [f2] "I" (f2) + : "memory"); +} + +static __always_inline void fpu_ld(unsigned short fpr, freg_t *reg) +{ + instrument_read(reg, sizeof(*reg)); + asm volatile("ld %[fpr],%[reg]\n" + : + : [fpr] "I" (fpr), [reg] "Q" (reg->ui) + : "memory"); +} + +static __always_inline void fpu_ldgr(u8 f1, u32 val) +{ + asm volatile("ldgr %[f1],%[val]\n" + : + : [f1] "I" (f1), [val] "d" (val) + : "memory"); +} + +static __always_inline void fpu_lfpc(unsigned int *fpc) +{ + instrument_read(fpc, sizeof(*fpc)); + asm volatile("lfpc %[fpc]" + : + : [fpc] "Q" (*fpc) + : "memory"); +} + +/** + * fpu_lfpc_safe - Load floating point control register safely. + * @fpc: new value for floating point control register + * + * Load floating point control register. This may lead to an exception, + * since a saved value may have been modified by user space (ptrace, + * signal return, kvm registers) to an invalid value. In such a case + * set the floating point control register to zero. + */ +static inline void fpu_lfpc_safe(unsigned int *fpc) +{ + u32 tmp; + + instrument_read(fpc, sizeof(*fpc)); + asm volatile("\n" + "0: lfpc %[fpc]\n" + "1: nopr %%r7\n" + ".pushsection .fixup, \"ax\"\n" + "2: lghi %[tmp],0\n" + " sfpc %[tmp]\n" + " jg 1b\n" + ".popsection\n" + EX_TABLE(1b, 2b) + : [tmp] "=d" (tmp) + : [fpc] "Q" (*fpc) + : "memory"); +} + +static __always_inline void fpu_std(unsigned short fpr, freg_t *reg) +{ + instrument_write(reg, sizeof(*reg)); + asm volatile("std %[fpr],%[reg]\n" + : [reg] "=Q" (reg->ui) + : [fpr] "I" (fpr) + : "memory"); +} + +static __always_inline void fpu_sfpc(unsigned int fpc) +{ + asm volatile("sfpc %[fpc]" + : + : [fpc] "d" (fpc) + : "memory"); +} + +static __always_inline void fpu_stfpc(unsigned int *fpc) +{ + instrument_write(fpc, sizeof(*fpc)); + asm volatile("stfpc %[fpc]" + : [fpc] "=Q" (*fpc) + : + : "memory"); +} + +static __always_inline void fpu_vab(u8 v1, u8 v2, u8 v3) +{ + asm volatile("VAB %[v1],%[v2],%[v3]" + : + : [v1] "I" (v1), [v2] "I" (v2), [v3] "I" (v3) + : "memory"); +} + +static __always_inline void fpu_vcksm(u8 v1, u8 v2, u8 v3) +{ + asm volatile("VCKSM %[v1],%[v2],%[v3]" + : + : [v1] "I" (v1), [v2] "I" (v2), [v3] "I" (v3) + : "memory"); +} + +static __always_inline void fpu_vesravb(u8 v1, u8 v2, u8 v3) +{ + asm volatile("VESRAVB %[v1],%[v2],%[v3]" + : + : [v1] "I" (v1), [v2] "I" (v2), [v3] "I" (v3) + : "memory"); +} + +static __always_inline void fpu_vgfmag(u8 v1, u8 v2, u8 v3, u8 v4) +{ + asm volatile("VGFMAG %[v1],%[v2],%[v3],%[v4]" + : + : [v1] "I" (v1), [v2] "I" (v2), [v3] "I" (v3), [v4] "I" (v4) + : "memory"); +} + +static __always_inline void fpu_vgfmg(u8 v1, u8 v2, u8 v3) +{ + asm volatile("VGFMG %[v1],%[v2],%[v3]" + : + : [v1] "I" (v1), [v2] "I" (v2), [v3] "I" (v3) + : "memory"); +} + +#ifdef CONFIG_CC_IS_CLANG + +static __always_inline void fpu_vl(u8 v1, const void *vxr) +{ + instrument_read(vxr, sizeof(__vector128)); + asm volatile("\n" + " la 1,%[vxr]\n" + " VL %[v1],0,,1\n" + : + : [vxr] "R" (*(__vector128 *)vxr), + [v1] "I" (v1) + : "memory", "1"); +} + +#else /* CONFIG_CC_IS_CLANG */ + +static __always_inline void fpu_vl(u8 v1, const void *vxr) +{ + instrument_read(vxr, sizeof(__vector128)); + asm volatile("VL %[v1],%O[vxr],,%R[vxr]\n" + : + : [vxr] "Q" (*(__vector128 *)vxr), + [v1] "I" (v1) + : "memory"); +} + +#endif /* CONFIG_CC_IS_CLANG */ + +static __always_inline void fpu_vleib(u8 v, s16 val, u8 index) +{ + asm volatile("VLEIB %[v],%[val],%[index]" + : + : [v] "I" (v), [val] "K" (val), [index] "I" (index) + : "memory"); +} + +static __always_inline void fpu_vleig(u8 v, s16 val, u8 index) +{ + asm volatile("VLEIG %[v],%[val],%[index]" + : + : [v] "I" (v), [val] "K" (val), [index] "I" (index) + : "memory"); +} + +static __always_inline u64 fpu_vlgvf(u8 v, u16 index) +{ + u64 val; + + asm volatile("VLGVF %[val],%[v],%[index]" + : [val] "=d" (val) + : [v] "I" (v), [index] "L" (index) + : "memory"); + return val; +} + +#ifdef CONFIG_CC_IS_CLANG + +static __always_inline void fpu_vll(u8 v1, u32 index, const void *vxr) +{ + unsigned int size; + + size = min(index + 1, sizeof(__vector128)); + instrument_read(vxr, size); + asm volatile("\n" + " la 1,%[vxr]\n" + " VLL %[v1],%[index],0,1\n" + : + : [vxr] "R" (*(u8 *)vxr), + [index] "d" (index), + [v1] "I" (v1) + : "memory", "1"); +} + +#else /* CONFIG_CC_IS_CLANG */ + +static __always_inline void fpu_vll(u8 v1, u32 index, const void *vxr) +{ + unsigned int size; + + size = min(index + 1, sizeof(__vector128)); + instrument_read(vxr, size); + asm volatile("VLL %[v1],%[index],%O[vxr],%R[vxr]\n" + : + : [vxr] "Q" (*(u8 *)vxr), + [index] "d" (index), + [v1] "I" (v1) + : "memory"); +} + +#endif /* CONFIG_CC_IS_CLANG */ + +#ifdef CONFIG_CC_IS_CLANG + +#define fpu_vlm(_v1, _v3, _vxrs) \ +({ \ + unsigned int size = ((_v3) - (_v1) + 1) * sizeof(__vector128); \ + struct { \ + __vector128 _v[(_v3) - (_v1) + 1]; \ + } *_v = (void *)(_vxrs); \ + \ + instrument_read(_v, size); \ + asm volatile("\n" \ + " la 1,%[vxrs]\n" \ + " VLM %[v1],%[v3],0,1\n" \ + : \ + : [vxrs] "R" (*_v), \ + [v1] "I" (_v1), [v3] "I" (_v3) \ + : "memory", "1"); \ + (_v3) - (_v1) + 1; \ +}) + +#else /* CONFIG_CC_IS_CLANG */ + +#define fpu_vlm(_v1, _v3, _vxrs) \ +({ \ + unsigned int size = ((_v3) - (_v1) + 1) * sizeof(__vector128); \ + struct { \ + __vector128 _v[(_v3) - (_v1) + 1]; \ + } *_v = (void *)(_vxrs); \ + \ + instrument_read(_v, size); \ + asm volatile("VLM %[v1],%[v3],%O[vxrs],%R[vxrs]\n" \ + : \ + : [vxrs] "Q" (*_v), \ + [v1] "I" (_v1), [v3] "I" (_v3) \ + : "memory"); \ + (_v3) - (_v1) + 1; \ +}) + +#endif /* CONFIG_CC_IS_CLANG */ + +static __always_inline void fpu_vlr(u8 v1, u8 v2) +{ + asm volatile("VLR %[v1],%[v2]" + : + : [v1] "I" (v1), [v2] "I" (v2) + : "memory"); +} + +static __always_inline void fpu_vlvgf(u8 v, u32 val, u16 index) +{ + asm volatile("VLVGF %[v],%[val],%[index]" + : + : [v] "I" (v), [val] "d" (val), [index] "L" (index) + : "memory"); +} + +static __always_inline void fpu_vn(u8 v1, u8 v2, u8 v3) +{ + asm volatile("VN %[v1],%[v2],%[v3]" + : + : [v1] "I" (v1), [v2] "I" (v2), [v3] "I" (v3) + : "memory"); +} + +static __always_inline void fpu_vperm(u8 v1, u8 v2, u8 v3, u8 v4) +{ + asm volatile("VPERM %[v1],%[v2],%[v3],%[v4]" + : + : [v1] "I" (v1), [v2] "I" (v2), [v3] "I" (v3), [v4] "I" (v4) + : "memory"); +} + +static __always_inline void fpu_vrepib(u8 v1, s16 i2) +{ + asm volatile("VREPIB %[v1],%[i2]" + : + : [v1] "I" (v1), [i2] "K" (i2) + : "memory"); +} + +static __always_inline void fpu_vsrlb(u8 v1, u8 v2, u8 v3) +{ + asm volatile("VSRLB %[v1],%[v2],%[v3]" + : + : [v1] "I" (v1), [v2] "I" (v2), [v3] "I" (v3) + : "memory"); +} + +#ifdef CONFIG_CC_IS_CLANG + +static __always_inline void fpu_vst(u8 v1, const void *vxr) +{ + instrument_write(vxr, sizeof(__vector128)); + asm volatile("\n" + " la 1,%[vxr]\n" + " VST %[v1],0,,1\n" + : [vxr] "=R" (*(__vector128 *)vxr) + : [v1] "I" (v1) + : "memory", "1"); +} + +#else /* CONFIG_CC_IS_CLANG */ + +static __always_inline void fpu_vst(u8 v1, const void *vxr) +{ + instrument_write(vxr, sizeof(__vector128)); + asm volatile("VST %[v1],%O[vxr],,%R[vxr]\n" + : [vxr] "=Q" (*(__vector128 *)vxr) + : [v1] "I" (v1) + : "memory"); +} + +#endif /* CONFIG_CC_IS_CLANG */ + +#ifdef CONFIG_CC_IS_CLANG + +static __always_inline void fpu_vstl(u8 v1, u32 index, const void *vxr) +{ + unsigned int size; + + size = min(index + 1, sizeof(__vector128)); + instrument_write(vxr, size); + asm volatile("\n" + " la 1,%[vxr]\n" + " VSTL %[v1],%[index],0,1\n" + : [vxr] "=R" (*(u8 *)vxr) + : [index] "d" (index), [v1] "I" (v1) + : "memory", "1"); +} + +#else /* CONFIG_CC_IS_CLANG */ + +static __always_inline void fpu_vstl(u8 v1, u32 index, const void *vxr) +{ + unsigned int size; + + size = min(index + 1, sizeof(__vector128)); + instrument_write(vxr, size); + asm volatile("VSTL %[v1],%[index],%O[vxr],%R[vxr]\n" + : [vxr] "=Q" (*(u8 *)vxr) + : [index] "d" (index), [v1] "I" (v1) + : "memory"); +} + +#endif /* CONFIG_CC_IS_CLANG */ + +#ifdef CONFIG_CC_IS_CLANG + +#define fpu_vstm(_v1, _v3, _vxrs) \ +({ \ + unsigned int size = ((_v3) - (_v1) + 1) * sizeof(__vector128); \ + struct { \ + __vector128 _v[(_v3) - (_v1) + 1]; \ + } *_v = (void *)(_vxrs); \ + \ + instrument_write(_v, size); \ + asm volatile("\n" \ + " la 1,%[vxrs]\n" \ + " VSTM %[v1],%[v3],0,1\n" \ + : [vxrs] "=R" (*_v) \ + : [v1] "I" (_v1), [v3] "I" (_v3) \ + : "memory", "1"); \ + (_v3) - (_v1) + 1; \ +}) + +#else /* CONFIG_CC_IS_CLANG */ + +#define fpu_vstm(_v1, _v3, _vxrs) \ +({ \ + unsigned int size = ((_v3) - (_v1) + 1) * sizeof(__vector128); \ + struct { \ + __vector128 _v[(_v3) - (_v1) + 1]; \ + } *_v = (void *)(_vxrs); \ + \ + instrument_write(_v, size); \ + asm volatile("VSTM %[v1],%[v3],%O[vxrs],%R[vxrs]\n" \ + : [vxrs] "=Q" (*_v) \ + : [v1] "I" (_v1), [v3] "I" (_v3) \ + : "memory"); \ + (_v3) - (_v1) + 1; \ +}) + +#endif /* CONFIG_CC_IS_CLANG */ + +static __always_inline void fpu_vupllf(u8 v1, u8 v2) +{ + asm volatile("VUPLLF %[v1],%[v2]" + : + : [v1] "I" (v1), [v2] "I" (v2) + : "memory"); +} + +static __always_inline void fpu_vx(u8 v1, u8 v2, u8 v3) +{ + asm volatile("VX %[v1],%[v2],%[v3]" + : + : [v1] "I" (v1), [v2] "I" (v2), [v3] "I" (v3) + : "memory"); +} + +static __always_inline void fpu_vzero(u8 v) +{ + asm volatile("VZERO %[v]" + : + : [v] "I" (v) + : "memory"); +} + +#endif /* __ASSEMBLY__ */ +#endif /* __ASM_S390_FPU_INSN_H */ diff --git a/arch/s390/include/asm/fpu-types.h b/arch/s390/include/asm/fpu-types.h new file mode 100644 index 000000000000..8d58d5a95399 --- /dev/null +++ b/arch/s390/include/asm/fpu-types.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * FPU data structures + * + * Copyright IBM Corp. 2015 + * Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com> + */ + +#ifndef _ASM_S390_FPU_TYPES_H +#define _ASM_S390_FPU_TYPES_H + +#include <asm/sigcontext.h> + +struct fpu { + u32 fpc; + __vector128 vxrs[__NUM_VXRS] __aligned(8); +}; + +struct kernel_fpu_hdr { + int mask; + u32 fpc; +}; + +struct kernel_fpu { + struct kernel_fpu_hdr hdr; + __vector128 vxrs[] __aligned(8); +}; + +#define KERNEL_FPU_STRUCT(vxr_size) \ +struct kernel_fpu_##vxr_size { \ + struct kernel_fpu_hdr hdr; \ + __vector128 vxrs[vxr_size] __aligned(8); \ +} + +KERNEL_FPU_STRUCT(8); +KERNEL_FPU_STRUCT(16); +KERNEL_FPU_STRUCT(32); + +#define DECLARE_KERNEL_FPU_ONSTACK(vxr_size, name) \ + struct kernel_fpu_##vxr_size name __uninitialized + +#define DECLARE_KERNEL_FPU_ONSTACK8(name) \ + DECLARE_KERNEL_FPU_ONSTACK(8, name) + +#define DECLARE_KERNEL_FPU_ONSTACK16(name) \ + DECLARE_KERNEL_FPU_ONSTACK(16, name) + +#define DECLARE_KERNEL_FPU_ONSTACK32(name) \ + DECLARE_KERNEL_FPU_ONSTACK(32, name) + +#endif /* _ASM_S390_FPU_TYPES_H */ diff --git a/arch/s390/include/asm/fpu.h b/arch/s390/include/asm/fpu.h new file mode 100644 index 000000000000..c84cb33913e2 --- /dev/null +++ b/arch/s390/include/asm/fpu.h @@ -0,0 +1,295 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * In-kernel FPU support functions + * + * + * Consider these guidelines before using in-kernel FPU functions: + * + * 1. Use kernel_fpu_begin() and kernel_fpu_end() to enclose all in-kernel + * use of floating-point or vector registers and instructions. + * + * 2. For kernel_fpu_begin(), specify the vector register range you want to + * use with the KERNEL_VXR_* constants. Consider these usage guidelines: + * + * a) If your function typically runs in process-context, use the lower + * half of the vector registers, for example, specify KERNEL_VXR_LOW. + * b) If your function typically runs in soft-irq or hard-irq context, + * prefer using the upper half of the vector registers, for example, + * specify KERNEL_VXR_HIGH. + * + * If you adhere to these guidelines, an interrupted process context + * does not require to save and restore vector registers because of + * disjoint register ranges. + * + * Also note that the __kernel_fpu_begin()/__kernel_fpu_end() functions + * includes logic to save and restore up to 16 vector registers at once. + * + * 3. You can nest kernel_fpu_begin()/kernel_fpu_end() by using different + * struct kernel_fpu states. Vector registers that are in use by outer + * levels are saved and restored. You can minimize the save and restore + * effort by choosing disjoint vector register ranges. + * + * 5. To use vector floating-point instructions, specify the KERNEL_FPC + * flag to save and restore floating-point controls in addition to any + * vector register range. + * + * 6. To use floating-point registers and instructions only, specify the + * KERNEL_FPR flag. This flag triggers a save and restore of vector + * registers V0 to V15 and floating-point controls. + * + * Copyright IBM Corp. 2015 + * Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com> + */ + +#ifndef _ASM_S390_FPU_H +#define _ASM_S390_FPU_H + +#include <linux/processor.h> +#include <linux/preempt.h> +#include <linux/string.h> +#include <linux/sched.h> +#include <asm/sigcontext.h> +#include <asm/fpu-types.h> +#include <asm/fpu-insn.h> +#include <asm/facility.h> + +static inline bool cpu_has_vx(void) +{ + return likely(test_facility(129)); +} + +enum { + KERNEL_FPC_BIT = 0, + KERNEL_VXR_V0V7_BIT, + KERNEL_VXR_V8V15_BIT, + KERNEL_VXR_V16V23_BIT, + KERNEL_VXR_V24V31_BIT, +}; + +#define KERNEL_FPC BIT(KERNEL_FPC_BIT) +#define KERNEL_VXR_V0V7 BIT(KERNEL_VXR_V0V7_BIT) +#define KERNEL_VXR_V8V15 BIT(KERNEL_VXR_V8V15_BIT) +#define KERNEL_VXR_V16V23 BIT(KERNEL_VXR_V16V23_BIT) +#define KERNEL_VXR_V24V31 BIT(KERNEL_VXR_V24V31_BIT) + +#define KERNEL_VXR_LOW (KERNEL_VXR_V0V7 | KERNEL_VXR_V8V15) +#define KERNEL_VXR_MID (KERNEL_VXR_V8V15 | KERNEL_VXR_V16V23) +#define KERNEL_VXR_HIGH (KERNEL_VXR_V16V23 | KERNEL_VXR_V24V31) + +#define KERNEL_VXR (KERNEL_VXR_LOW | KERNEL_VXR_HIGH) +#define KERNEL_FPR (KERNEL_FPC | KERNEL_VXR_LOW) + +void load_fpu_state(struct fpu *state, int flags); +void save_fpu_state(struct fpu *state, int flags); +void __kernel_fpu_begin(struct kernel_fpu *state, int flags); +void __kernel_fpu_end(struct kernel_fpu *state, int flags); + +static __always_inline void save_vx_regs(__vector128 *vxrs) +{ + fpu_vstm(0, 15, &vxrs[0]); + fpu_vstm(16, 31, &vxrs[16]); +} + +static __always_inline void load_vx_regs(__vector128 *vxrs) +{ + fpu_vlm(0, 15, &vxrs[0]); + fpu_vlm(16, 31, &vxrs[16]); +} + +static __always_inline void __save_fp_regs(freg_t *fprs, unsigned int offset) +{ + fpu_std(0, &fprs[0 * offset]); + fpu_std(1, &fprs[1 * offset]); + fpu_std(2, &fprs[2 * offset]); + fpu_std(3, &fprs[3 * offset]); + fpu_std(4, &fprs[4 * offset]); + fpu_std(5, &fprs[5 * offset]); + fpu_std(6, &fprs[6 * offset]); + fpu_std(7, &fprs[7 * offset]); + fpu_std(8, &fprs[8 * offset]); + fpu_std(9, &fprs[9 * offset]); + fpu_std(10, &fprs[10 * offset]); + fpu_std(11, &fprs[11 * offset]); + fpu_std(12, &fprs[12 * offset]); + fpu_std(13, &fprs[13 * offset]); + fpu_std(14, &fprs[14 * offset]); + fpu_std(15, &fprs[15 * offset]); +} + +static __always_inline void __load_fp_regs(freg_t *fprs, unsigned int offset) +{ + fpu_ld(0, &fprs[0 * offset]); + fpu_ld(1, &fprs[1 * offset]); + fpu_ld(2, &fprs[2 * offset]); + fpu_ld(3, &fprs[3 * offset]); + fpu_ld(4, &fprs[4 * offset]); + fpu_ld(5, &fprs[5 * offset]); + fpu_ld(6, &fprs[6 * offset]); + fpu_ld(7, &fprs[7 * offset]); + fpu_ld(8, &fprs[8 * offset]); + fpu_ld(9, &fprs[9 * offset]); + fpu_ld(10, &fprs[10 * offset]); + fpu_ld(11, &fprs[11 * offset]); + fpu_ld(12, &fprs[12 * offset]); + fpu_ld(13, &fprs[13 * offset]); + fpu_ld(14, &fprs[14 * offset]); + fpu_ld(15, &fprs[15 * offset]); +} + +static __always_inline void save_fp_regs(freg_t *fprs) +{ + __save_fp_regs(fprs, sizeof(freg_t) / sizeof(freg_t)); +} + +static __always_inline void load_fp_regs(freg_t *fprs) +{ + __load_fp_regs(fprs, sizeof(freg_t) / sizeof(freg_t)); +} + +static __always_inline void save_fp_regs_vx(__vector128 *vxrs) +{ + freg_t *fprs = (freg_t *)&vxrs[0].high; + + __save_fp_regs(fprs, sizeof(__vector128) / sizeof(freg_t)); +} + +static __always_inline void load_fp_regs_vx(__vector128 *vxrs) +{ + freg_t *fprs = (freg_t *)&vxrs[0].high; + + __load_fp_regs(fprs, sizeof(__vector128) / sizeof(freg_t)); +} + +static inline void load_user_fpu_regs(void) +{ + struct thread_struct *thread = ¤t->thread; + + if (!thread->ufpu_flags) + return; + load_fpu_state(&thread->ufpu, thread->ufpu_flags); + thread->ufpu_flags = 0; +} + +static __always_inline void __save_user_fpu_regs(struct thread_struct *thread, int flags) +{ + save_fpu_state(&thread->ufpu, flags); + __atomic_or(flags, &thread->ufpu_flags); +} + +static inline void save_user_fpu_regs(void) +{ + struct thread_struct *thread = ¤t->thread; + int mask, flags; + + mask = __atomic_or(KERNEL_FPC | KERNEL_VXR, &thread->kfpu_flags); + flags = ~READ_ONCE(thread->ufpu_flags) & (KERNEL_FPC | KERNEL_VXR); + if (flags) + __save_user_fpu_regs(thread, flags); + barrier(); + WRITE_ONCE(thread->kfpu_flags, mask); +} + +static __always_inline void _kernel_fpu_begin(struct kernel_fpu *state, int flags) +{ + struct thread_struct *thread = ¤t->thread; + int mask, uflags; + + mask = __atomic_or(flags, &thread->kfpu_flags); + state->hdr.mask = mask; + uflags = READ_ONCE(thread->ufpu_flags); + if ((uflags & flags) != flags) + __save_user_fpu_regs(thread, ~uflags & flags); + if (mask & flags) + __kernel_fpu_begin(state, flags); +} + +static __always_inline void _kernel_fpu_end(struct kernel_fpu *state, int flags) +{ + int mask = state->hdr.mask; + + if (mask & flags) + __kernel_fpu_end(state, flags); + barrier(); + WRITE_ONCE(current->thread.kfpu_flags, mask); +} + +void __kernel_fpu_invalid_size(void); + +static __always_inline void kernel_fpu_check_size(int flags, unsigned int size) +{ + unsigned int cnt = 0; + + if (flags & KERNEL_VXR_V0V7) + cnt += 8; + if (flags & KERNEL_VXR_V8V15) + cnt += 8; + if (flags & KERNEL_VXR_V16V23) + cnt += 8; + if (flags & KERNEL_VXR_V24V31) + cnt += 8; + if (cnt != size) + __kernel_fpu_invalid_size(); +} + +#define kernel_fpu_begin(state, flags) \ +{ \ + typeof(state) s = (state); \ + int _flags = (flags); \ + \ + kernel_fpu_check_size(_flags, ARRAY_SIZE(s->vxrs)); \ + _kernel_fpu_begin((struct kernel_fpu *)s, _flags); \ +} + +#define kernel_fpu_end(state, flags) \ +{ \ + typeof(state) s = (state); \ + int _flags = (flags); \ + \ + kernel_fpu_check_size(_flags, ARRAY_SIZE(s->vxrs)); \ + _kernel_fpu_end((struct kernel_fpu *)s, _flags); \ +} + +static inline void save_kernel_fpu_regs(struct thread_struct *thread) +{ + if (!thread->kfpu_flags) + return; + save_fpu_state(&thread->kfpu, thread->kfpu_flags); +} + +static inline void restore_kernel_fpu_regs(struct thread_struct *thread) +{ + if (!thread->kfpu_flags) + return; + load_fpu_state(&thread->kfpu, thread->kfpu_flags); +} + +static inline void convert_vx_to_fp(freg_t *fprs, __vector128 *vxrs) +{ + int i; + + for (i = 0; i < __NUM_FPRS; i++) + fprs[i].ui = vxrs[i].high; +} + +static inline void convert_fp_to_vx(__vector128 *vxrs, freg_t *fprs) +{ + int i; + + for (i = 0; i < __NUM_FPRS; i++) + vxrs[i].high = fprs[i].ui; +} + +static inline void fpregs_store(_s390_fp_regs *fpregs, struct fpu *fpu) +{ + fpregs->pad = 0; + fpregs->fpc = fpu->fpc; + convert_vx_to_fp((freg_t *)&fpregs->fprs, fpu->vxrs); +} + +static inline void fpregs_load(_s390_fp_regs *fpregs, struct fpu *fpu) +{ + fpu->fpc = fpregs->fpc; + convert_fp_to_vx(fpu->vxrs, (freg_t *)&fpregs->fprs); +} + +#endif /* _ASM_S390_FPU_H */ diff --git a/arch/s390/include/asm/fpu/api.h b/arch/s390/include/asm/fpu/api.h deleted file mode 100644 index d6ca8bc6ca68..000000000000 --- a/arch/s390/include/asm/fpu/api.h +++ /dev/null @@ -1,126 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * In-kernel FPU support functions - * - * - * Consider these guidelines before using in-kernel FPU functions: - * - * 1. Use kernel_fpu_begin() and kernel_fpu_end() to enclose all in-kernel - * use of floating-point or vector registers and instructions. - * - * 2. For kernel_fpu_begin(), specify the vector register range you want to - * use with the KERNEL_VXR_* constants. Consider these usage guidelines: - * - * a) If your function typically runs in process-context, use the lower - * half of the vector registers, for example, specify KERNEL_VXR_LOW. - * b) If your function typically runs in soft-irq or hard-irq context, - * prefer using the upper half of the vector registers, for example, - * specify KERNEL_VXR_HIGH. - * - * If you adhere to these guidelines, an interrupted process context - * does not require to save and restore vector registers because of - * disjoint register ranges. - * - * Also note that the __kernel_fpu_begin()/__kernel_fpu_end() functions - * includes logic to save and restore up to 16 vector registers at once. - * - * 3. You can nest kernel_fpu_begin()/kernel_fpu_end() by using different - * struct kernel_fpu states. Vector registers that are in use by outer - * levels are saved and restored. You can minimize the save and restore - * effort by choosing disjoint vector register ranges. - * - * 5. To use vector floating-point instructions, specify the KERNEL_FPC - * flag to save and restore floating-point controls in addition to any - * vector register range. - * - * 6. To use floating-point registers and instructions only, specify the - * KERNEL_FPR flag. This flag triggers a save and restore of vector - * registers V0 to V15 and floating-point controls. - * - * Copyright IBM Corp. 2015 - * Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com> - */ - -#ifndef _ASM_S390_FPU_API_H -#define _ASM_S390_FPU_API_H - -#include <linux/preempt.h> -#include <asm/asm-extable.h> -#include <asm/fpu/internal.h> - -void save_fpu_regs(void); -void load_fpu_regs(void); -void __load_fpu_regs(void); - -/** - * sfpc_safe - Set floating point control register safely. - * @fpc: new value for floating point control register - * - * Set floating point control register. This may lead to an exception, - * since a saved value may have been modified by user space (ptrace, - * signal return, kvm registers) to an invalid value. In such a case - * set the floating point control register to zero. - */ -static inline void sfpc_safe(u32 fpc) -{ - asm volatile("\n" - "0: sfpc %[fpc]\n" - "1: nopr %%r7\n" - ".pushsection .fixup, \"ax\"\n" - "2: lghi %[fpc],0\n" - " jg 0b\n" - ".popsection\n" - EX_TABLE(1b, 2b) - : [fpc] "+d" (fpc) - : : "memory"); -} - -#define KERNEL_FPC 1 -#define KERNEL_VXR_V0V7 2 -#define KERNEL_VXR_V8V15 4 -#define KERNEL_VXR_V16V23 8 -#define KERNEL_VXR_V24V31 16 - -#define KERNEL_VXR_LOW (KERNEL_VXR_V0V7|KERNEL_VXR_V8V15) -#define KERNEL_VXR_MID (KERNEL_VXR_V8V15|KERNEL_VXR_V16V23) -#define KERNEL_VXR_HIGH (KERNEL_VXR_V16V23|KERNEL_VXR_V24V31) - -#define KERNEL_VXR (KERNEL_VXR_LOW|KERNEL_VXR_HIGH) -#define KERNEL_FPR (KERNEL_FPC|KERNEL_VXR_LOW) - -struct kernel_fpu; - -/* - * Note the functions below must be called with preemption disabled. - * Do not enable preemption before calling __kernel_fpu_end() to prevent - * an corruption of an existing kernel FPU state. - * - * Prefer using the kernel_fpu_begin()/kernel_fpu_end() pair of functions. - */ -void __kernel_fpu_begin(struct kernel_fpu *state, u32 flags); -void __kernel_fpu_end(struct kernel_fpu *state, u32 flags); - - -static inline void kernel_fpu_begin(struct kernel_fpu *state, u32 flags) -{ - preempt_disable(); - state->mask = S390_lowcore.fpu_flags; - if (!test_cpu_flag(CIF_FPU)) - /* Save user space FPU state and register contents */ - save_fpu_regs(); - else if (state->mask & flags) - /* Save FPU/vector register in-use by the kernel */ - __kernel_fpu_begin(state, flags); - S390_lowcore.fpu_flags |= flags; -} - -static inline void kernel_fpu_end(struct kernel_fpu *state, u32 flags) -{ - S390_lowcore.fpu_flags = state->mask; - if (state->mask & flags) - /* Restore FPU/vector register in-use by the kernel */ - __kernel_fpu_end(state, flags); - preempt_enable(); -} - -#endif /* _ASM_S390_FPU_API_H */ diff --git a/arch/s390/include/asm/fpu/internal.h b/arch/s390/include/asm/fpu/internal.h deleted file mode 100644 index d511c4cf5afb..000000000000 --- a/arch/s390/include/asm/fpu/internal.h +++ /dev/null @@ -1,67 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * FPU state and register content conversion primitives - * - * Copyright IBM Corp. 2015 - * Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com> - */ - -#ifndef _ASM_S390_FPU_INTERNAL_H -#define _ASM_S390_FPU_INTERNAL_H - -#include <linux/string.h> -#include <asm/facility.h> -#include <asm/fpu/types.h> - -static inline bool cpu_has_vx(void) -{ - return likely(test_facility(129)); -} - -static inline void save_vx_regs(__vector128 *vxrs) -{ - asm volatile( - " la 1,%0\n" - " .word 0xe70f,0x1000,0x003e\n" /* vstm 0,15,0(1) */ - " .word 0xe70f,0x1100,0x0c3e\n" /* vstm 16,31,256(1) */ - : "=Q" (*(struct vx_array *) vxrs) : : "1"); -} - -static inline void convert_vx_to_fp(freg_t *fprs, __vector128 *vxrs) -{ - int i; - - for (i = 0; i < __NUM_FPRS; i++) - fprs[i].ui = vxrs[i].high; -} - -static inline void convert_fp_to_vx(__vector128 *vxrs, freg_t *fprs) -{ - int i; - - for (i = 0; i < __NUM_FPRS; i++) - vxrs[i].high = fprs[i].ui; -} - -static inline void fpregs_store(_s390_fp_regs *fpregs, struct fpu *fpu) -{ - fpregs->pad = 0; - fpregs->fpc = fpu->fpc; - if (cpu_has_vx()) - convert_vx_to_fp((freg_t *)&fpregs->fprs, fpu->vxrs); - else - memcpy((freg_t *)&fpregs->fprs, fpu->fprs, - sizeof(fpregs->fprs)); -} - -static inline void fpregs_load(_s390_fp_regs *fpregs, struct fpu *fpu) -{ - fpu->fpc = fpregs->fpc; - if (cpu_has_vx()) - convert_fp_to_vx(fpu->vxrs, (freg_t *)&fpregs->fprs); - else - memcpy(fpu->fprs, (freg_t *)&fpregs->fprs, - sizeof(fpregs->fprs)); -} - -#endif /* _ASM_S390_FPU_INTERNAL_H */ diff --git a/arch/s390/include/asm/fpu/types.h b/arch/s390/include/asm/fpu/types.h deleted file mode 100644 index d889e9436865..000000000000 --- a/arch/s390/include/asm/fpu/types.h +++ /dev/null @@ -1,38 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * FPU data structures - * - * Copyright IBM Corp. 2015 - * Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com> - */ - -#ifndef _ASM_S390_FPU_TYPES_H -#define _ASM_S390_FPU_TYPES_H - -#include <asm/sigcontext.h> - -struct fpu { - __u32 fpc; /* Floating-point control */ - void *regs; /* Pointer to the current save area */ - union { - /* Floating-point register save area */ - freg_t fprs[__NUM_FPRS]; - /* Vector register save area */ - __vector128 vxrs[__NUM_VXRS]; - }; -}; - -/* VX array structure for address operand constraints in inline assemblies */ -struct vx_array { __vector128 _[__NUM_VXRS]; }; - -/* In-kernel FPU state structure */ -struct kernel_fpu { - u32 mask; - u32 fpc; - union { - freg_t fprs[__NUM_FPRS]; - __vector128 vxrs[__NUM_VXRS]; - }; -}; - -#endif /* _ASM_S390_FPU_TYPES_H */ diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h index 52664105a473..95990461888f 100644 --- a/arch/s390/include/asm/kvm_host.h +++ b/arch/s390/include/asm/kvm_host.h @@ -23,7 +23,7 @@ #include <linux/mmu_notifier.h> #include <asm/debug.h> #include <asm/cpu.h> -#include <asm/fpu/api.h> +#include <asm/fpu.h> #include <asm/isc.h> #include <asm/guarded_storage.h> @@ -743,7 +743,6 @@ struct kvm_vcpu_arch { struct kvm_s390_sie_block *vsie_block; unsigned int host_acrs[NUM_ACRS]; struct gs_cb *host_gscb; - struct fpu host_fpregs; struct kvm_s390_local_interrupt local_int; struct hrtimer ckc_timer; struct kvm_s390_pgm_info pgm; @@ -765,6 +764,8 @@ struct kvm_vcpu_arch { __u64 cputm_start; bool gs_enabled; bool skey_enabled; + /* Indicator if the access registers have been loaded from guest */ + bool acrs_loaded; struct kvm_s390_pv_vcpu pv; union diag318_info diag318_info; }; diff --git a/arch/s390/include/asm/lowcore.h b/arch/s390/include/asm/lowcore.h index 5dc1b6345006..8c5f16857539 100644 --- a/arch/s390/include/asm/lowcore.h +++ b/arch/s390/include/asm/lowcore.h @@ -157,7 +157,7 @@ struct lowcore { __s32 preempt_count; /* 0x03a8 */ __u32 spinlock_lockval; /* 0x03ac */ __u32 spinlock_index; /* 0x03b0 */ - __u32 fpu_flags; /* 0x03b4 */ + __u8 pad_0x03b4[0x03b8-0x03b4]; /* 0x03b4 */ __u64 percpu_offset; /* 0x03b8 */ __u8 pad_0x03c0[0x03c8-0x03c0]; /* 0x03c0 */ __u64 machine_flags; /* 0x03c8 */ diff --git a/arch/s390/include/asm/pai.h b/arch/s390/include/asm/pai.h index 7d1888e3dee6..3f609565734b 100644 --- a/arch/s390/include/asm/pai.h +++ b/arch/s390/include/asm/pai.h @@ -16,7 +16,7 @@ struct qpaci_info_block { u64 header; struct { u64 : 8; - u64 num_cc : 8; /* # of supported crypto counters */ + u64 num_cc : 8; /* # of supported crypto counters */ u64 : 9; u64 num_nnpa : 7; /* # of supported NNPA counters */ u64 : 32; @@ -81,4 +81,5 @@ enum paievt_mode { PAI_MODE_COUNTING, }; +#define PAI_SAVE_AREA(x) ((x)->hw.event_base) #endif diff --git a/arch/s390/include/asm/pci.h b/arch/s390/include/asm/pci.h index e91cd6bbc330..30820a649e6e 100644 --- a/arch/s390/include/asm/pci.h +++ b/arch/s390/include/asm/pci.h @@ -122,6 +122,7 @@ struct zpci_dev { struct rcu_head rcu; struct hotplug_slot hotplug_slot; + struct mutex state_lock; /* protect state changes */ enum zpci_state state; u32 fid; /* function ID, used by sclp */ u32 fh; /* function handle, used by insn's */ @@ -142,7 +143,6 @@ struct zpci_dev { u8 reserved : 2; unsigned int devfn; /* DEVFN part of the RID*/ - struct mutex lock; u8 pfip[CLP_PFIP_NR_SEGMENTS]; /* pci function internal path */ u32 uid; /* user defined id */ u8 util_str[CLP_UTIL_STR_LEN]; /* utility string */ @@ -170,6 +170,7 @@ struct zpci_dev { u64 dma_mask; /* DMA address space mask */ /* Function measurement block */ + struct mutex fmb_lock; struct zpci_fmb *fmb; u16 fmb_update; /* update interval */ u16 fmb_length; diff --git a/arch/s390/include/asm/physmem_info.h b/arch/s390/include/asm/physmem_info.h index 9e41a74fce9a..e747b067f8db 100644 --- a/arch/s390/include/asm/physmem_info.h +++ b/arch/s390/include/asm/physmem_info.h @@ -22,6 +22,7 @@ enum reserved_range_type { RR_DECOMPRESSOR, RR_INITRD, RR_VMLINUX, + RR_RELOC, RR_AMODE31, RR_IPLREPORT, RR_CERT_COMP_LIST, diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h index c0b6e74d899a..7cf00cf8fb0b 100644 --- a/arch/s390/include/asm/processor.h +++ b/arch/s390/include/asm/processor.h @@ -15,13 +15,11 @@ #include <linux/bits.h> #define CIF_NOHZ_DELAY 2 /* delay HZ disable for a tick */ -#define CIF_FPU 3 /* restore FPU registers */ #define CIF_ENABLED_WAIT 5 /* in enabled wait state */ #define CIF_MCCK_GUEST 6 /* machine check happening in guest */ #define CIF_DEDICATED_CPU 7 /* this CPU is dedicated */ #define _CIF_NOHZ_DELAY BIT(CIF_NOHZ_DELAY) -#define _CIF_FPU BIT(CIF_FPU) #define _CIF_ENABLED_WAIT BIT(CIF_ENABLED_WAIT) #define _CIF_MCCK_GUEST BIT(CIF_MCCK_GUEST) #define _CIF_DEDICATED_CPU BIT(CIF_DEDICATED_CPU) @@ -33,13 +31,12 @@ #include <linux/cpumask.h> #include <linux/linkage.h> #include <linux/irqflags.h> +#include <asm/fpu-types.h> #include <asm/cpu.h> #include <asm/page.h> #include <asm/ptrace.h> #include <asm/setup.h> #include <asm/runtime_instr.h> -#include <asm/fpu/types.h> -#include <asm/fpu/internal.h> #include <asm/irqflags.h> typedef long (*sys_call_ptr_t)(struct pt_regs *regs); @@ -169,6 +166,8 @@ struct thread_struct { unsigned int gmap_write_flag; /* gmap fault write indication */ unsigned int gmap_int_code; /* int code of last gmap fault */ unsigned int gmap_pfault; /* signal of a pending guest pfault */ + int ufpu_flags; /* user fpu flags */ + int kfpu_flags; /* kernel fpu flags */ /* Per-thread information related to debugging */ struct per_regs per_user; /* User specified PER registers */ @@ -184,7 +183,8 @@ struct thread_struct { struct gs_cb *gs_cb; /* Current guarded storage cb */ struct gs_cb *gs_bc_cb; /* Broadcast guarded storage cb */ struct pgm_tdb trap_tdb; /* Transaction abort diagnose block */ - struct fpu fpu; /* FP and VX register save area */ + struct fpu ufpu; /* User FP and VX register save area */ + struct fpu kfpu; /* Kernel FP and VX register save area */ }; /* Flag to disable transactions. */ @@ -203,7 +203,6 @@ typedef struct thread_struct thread_struct; #define INIT_THREAD { \ .ksp = sizeof(init_stack) + (unsigned long) &init_stack, \ - .fpu.regs = (void *) init_task.thread.fpu.fprs, \ .last_break = 1, \ } diff --git a/arch/s390/include/asm/ptrace.h b/arch/s390/include/asm/ptrace.h index d28bf8fb2799..788bc4467445 100644 --- a/arch/s390/include/asm/ptrace.h +++ b/arch/s390/include/asm/ptrace.h @@ -203,6 +203,10 @@ static inline int test_and_clear_pt_regs_flag(struct pt_regs *regs, int flag) return ret; } +struct task_struct; + +void update_cr_regs(struct task_struct *task); + /* * These are defined as per linux/ptrace.h, which see. */ diff --git a/arch/s390/include/asm/stacktrace.h b/arch/s390/include/asm/stacktrace.h index 31ec4f545e03..433fde85b14e 100644 --- a/arch/s390/include/asm/stacktrace.h +++ b/arch/s390/include/asm/stacktrace.h @@ -4,7 +4,6 @@ #include <linux/uaccess.h> #include <linux/ptrace.h> -#include <asm/switch_to.h> struct stack_frame_user { unsigned long back_chain; diff --git a/arch/s390/include/asm/switch_to.h b/arch/s390/include/asm/switch_to.h deleted file mode 100644 index c61b2cc1a8a8..000000000000 --- a/arch/s390/include/asm/switch_to.h +++ /dev/null @@ -1,49 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Copyright IBM Corp. 1999, 2009 - * - * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> - */ - -#ifndef __ASM_SWITCH_TO_H -#define __ASM_SWITCH_TO_H - -#include <linux/thread_info.h> -#include <asm/fpu/api.h> -#include <asm/ptrace.h> -#include <asm/guarded_storage.h> - -extern struct task_struct *__switch_to(void *, void *); -extern void update_cr_regs(struct task_struct *task); - -static inline void save_access_regs(unsigned int *acrs) -{ - typedef struct { int _[NUM_ACRS]; } acrstype; - - asm volatile("stam 0,15,%0" : "=Q" (*(acrstype *)acrs)); -} - -static inline void restore_access_regs(unsigned int *acrs) -{ - typedef struct { int _[NUM_ACRS]; } acrstype; - - asm volatile("lam 0,15,%0" : : "Q" (*(acrstype *)acrs)); -} - -#define switch_to(prev, next, last) do { \ - /* save_fpu_regs() sets the CIF_FPU flag, which enforces \ - * a restore of the floating point / vector registers as \ - * soon as the next task returns to user space \ - */ \ - save_fpu_regs(); \ - save_access_regs(&prev->thread.acrs[0]); \ - save_ri_cb(prev->thread.ri_cb); \ - save_gs_cb(prev->thread.gs_cb); \ - update_cr_regs(next); \ - restore_access_regs(&next->thread.acrs[0]); \ - restore_ri_cb(next->thread.ri_cb, prev->thread.ri_cb); \ - restore_gs_cb(next->thread.gs_cb); \ - prev = __switch_to(prev, next); \ -} while (0) - -#endif /* __ASM_SWITCH_TO_H */ diff --git a/arch/s390/include/asm/vx-insn.h b/arch/s390/include/asm/vx-insn.h deleted file mode 100644 index 8c188f1c6d27..000000000000 --- a/arch/s390/include/asm/vx-insn.h +++ /dev/null @@ -1,19 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Support for Vector Instructions - * - * This wrapper header file allows to use the vector instruction macros in - * both assembler files as well as in inline assemblies in C files. - */ - -#ifndef __ASM_S390_VX_INSN_H -#define __ASM_S390_VX_INSN_H - -#include <asm/vx-insn-asm.h> - -#ifndef __ASSEMBLY__ - -asm(".include \"asm/vx-insn-asm.h\"\n"); - -#endif /* __ASSEMBLY__ */ -#endif /* __ASM_S390_VX_INSN_H */ diff --git a/arch/s390/kernel/cache.c b/arch/s390/kernel/cache.c index 56254fa06f99..4f2669030220 100644 --- a/arch/s390/kernel/cache.c +++ b/arch/s390/kernel/cache.c @@ -166,5 +166,6 @@ int populate_cache_leaves(unsigned int cpu) ci_leaf_init(this_leaf++, pvt, ctype, level, cpu); } } + this_cpu_ci->cpu_map_populated = true; return 0; } diff --git a/arch/s390/kernel/compat_signal.c b/arch/s390/kernel/compat_signal.c index f8fc6c25d051..1942e2a9f8db 100644 --- a/arch/s390/kernel/compat_signal.c +++ b/arch/s390/kernel/compat_signal.c @@ -24,12 +24,12 @@ #include <linux/tty.h> #include <linux/personality.h> #include <linux/binfmts.h> +#include <asm/access-regs.h> #include <asm/ucontext.h> #include <linux/uaccess.h> #include <asm/lowcore.h> -#include <asm/switch_to.h> #include <asm/vdso.h> -#include <asm/fpu/api.h> +#include <asm/fpu.h> #include "compat_linux.h" #include "compat_ptrace.h" #include "entry.h" @@ -56,7 +56,7 @@ typedef struct static void store_sigregs(void) { save_access_regs(current->thread.acrs); - save_fpu_regs(); + save_user_fpu_regs(); } /* Load registers after signal return */ @@ -79,7 +79,7 @@ static int save_sigregs32(struct pt_regs *regs, _sigregs32 __user *sregs) user_sregs.regs.gprs[i] = (__u32) regs->gprs[i]; memcpy(&user_sregs.regs.acrs, current->thread.acrs, sizeof(user_sregs.regs.acrs)); - fpregs_store((_s390_fp_regs *) &user_sregs.fpregs, ¤t->thread.fpu); + fpregs_store((_s390_fp_regs *) &user_sregs.fpregs, ¤t->thread.ufpu); if (__copy_to_user(sregs, &user_sregs, sizeof(_sigregs32))) return -EFAULT; return 0; @@ -113,7 +113,7 @@ static int restore_sigregs32(struct pt_regs *regs,_sigregs32 __user *sregs) regs->gprs[i] = (__u64) user_sregs.regs.gprs[i]; memcpy(¤t->thread.acrs, &user_sregs.regs.acrs, sizeof(current->thread.acrs)); - fpregs_load((_s390_fp_regs *) &user_sregs.fpregs, ¤t->thread.fpu); + fpregs_load((_s390_fp_regs *)&user_sregs.fpregs, ¤t->thread.ufpu); clear_pt_regs_flag(regs, PIF_SYSCALL); /* No longer in a system call */ return 0; @@ -136,11 +136,11 @@ static int save_sigregs_ext32(struct pt_regs *regs, /* Save vector registers to signal stack */ if (cpu_has_vx()) { for (i = 0; i < __NUM_VXRS_LOW; i++) - vxrs[i] = current->thread.fpu.vxrs[i].low; + vxrs[i] = current->thread.ufpu.vxrs[i].low; if (__copy_to_user(&sregs_ext->vxrs_low, vxrs, sizeof(sregs_ext->vxrs_low)) || __copy_to_user(&sregs_ext->vxrs_high, - current->thread.fpu.vxrs + __NUM_VXRS_LOW, + current->thread.ufpu.vxrs + __NUM_VXRS_LOW, sizeof(sregs_ext->vxrs_high))) return -EFAULT; } @@ -165,12 +165,12 @@ static int restore_sigregs_ext32(struct pt_regs *regs, if (cpu_has_vx()) { if (__copy_from_user(vxrs, &sregs_ext->vxrs_low, sizeof(sregs_ext->vxrs_low)) || - __copy_from_user(current->thread.fpu.vxrs + __NUM_VXRS_LOW, + __copy_from_user(current->thread.ufpu.vxrs + __NUM_VXRS_LOW, &sregs_ext->vxrs_high, sizeof(sregs_ext->vxrs_high))) return -EFAULT; for (i = 0; i < __NUM_VXRS_LOW; i++) - current->thread.fpu.vxrs[i].low = vxrs[i]; + current->thread.ufpu.vxrs[i].low = vxrs[i]; } return 0; } @@ -184,7 +184,7 @@ COMPAT_SYSCALL_DEFINE0(sigreturn) if (get_compat_sigset(&set, (compat_sigset_t __user *)frame->sc.oldmask)) goto badframe; set_current_blocked(&set); - save_fpu_regs(); + save_user_fpu_regs(); if (restore_sigregs32(regs, &frame->sregs)) goto badframe; if (restore_sigregs_ext32(regs, &frame->sregs_ext)) @@ -207,7 +207,7 @@ COMPAT_SYSCALL_DEFINE0(rt_sigreturn) set_current_blocked(&set); if (compat_restore_altstack(&frame->uc.uc_stack)) goto badframe; - save_fpu_regs(); + save_user_fpu_regs(); if (restore_sigregs32(regs, &frame->uc.uc_mcontext)) goto badframe; if (restore_sigregs_ext32(regs, &frame->uc.uc_mcontext_ext)) diff --git a/arch/s390/kernel/crash_dump.c b/arch/s390/kernel/crash_dump.c index 5c46c2659305..d09ebb6f5262 100644 --- a/arch/s390/kernel/crash_dump.c +++ b/arch/s390/kernel/crash_dump.c @@ -22,7 +22,7 @@ #include <asm/ipl.h> #include <asm/sclp.h> #include <asm/maccess.h> -#include <asm/fpu/api.h> +#include <asm/fpu.h> #define PTR_ADD(x, y) (((char *) (x)) + ((unsigned long) (y))) #define PTR_SUB(x, y) (((char *) (x)) - ((unsigned long) (y))) diff --git a/arch/s390/kernel/diag.c b/arch/s390/kernel/diag.c index 92fdc35f028c..8dee9aa0ec95 100644 --- a/arch/s390/kernel/diag.c +++ b/arch/s390/kernel/diag.c @@ -147,11 +147,40 @@ void notrace diag_stat_inc_norecursion(enum diag_stat_enum nr) EXPORT_SYMBOL(diag_stat_inc_norecursion); /* + * Diagnose 0c: Pseudo Timer + */ +void diag0c(struct hypfs_diag0c_entry *data) +{ + diag_stat_inc(DIAG_STAT_X00C); + diag_amode31_ops.diag0c(virt_to_phys(data)); +} + +/* * Diagnose 14: Input spool file manipulation + * + * The subcode parameter determines the type of the first parameter rx. + * Currently used are the following 3 subcommands: + * 0x0: Read the Next Spool File Buffer (Data Record) + * 0x28: Position a Spool File to the Designated Record + * 0xfff: Retrieve Next File Descriptor + * + * For subcommands 0x0 and 0xfff, the value of the first parameter is + * a virtual address of a memory buffer and needs virtual to physical + * address translation. For other subcommands the rx parameter is not + * a virtual address. */ int diag14(unsigned long rx, unsigned long ry1, unsigned long subcode) { diag_stat_inc(DIAG_STAT_X014); + switch (subcode) { + case 0x0: + case 0xfff: + rx = virt_to_phys((void *)rx); + break; + default: + /* Do nothing */ + break; + } return diag_amode31_ops.diag14(rx, ry1, subcode); } EXPORT_SYMBOL(diag14); @@ -265,6 +294,6 @@ EXPORT_SYMBOL(diag224); int diag26c(void *req, void *resp, enum diag26c_sc subcode) { diag_stat_inc(DIAG_STAT_X26C); - return diag_amode31_ops.diag26c(req, resp, subcode); + return diag_amode31_ops.diag26c(virt_to_phys(req), virt_to_phys(resp), subcode); } EXPORT_SYMBOL(diag26c); diff --git a/arch/s390/kernel/early.c b/arch/s390/kernel/early.c index 2345ea332b97..c666271433fb 100644 --- a/arch/s390/kernel/early.c +++ b/arch/s390/kernel/early.c @@ -19,8 +19,10 @@ #include <linux/kernel.h> #include <asm/asm-extable.h> #include <linux/memblock.h> +#include <asm/access-regs.h> #include <asm/diag.h> #include <asm/ebcdic.h> +#include <asm/fpu.h> #include <asm/ipl.h> #include <asm/lowcore.h> #include <asm/processor.h> @@ -31,7 +33,6 @@ #include <asm/sclp.h> #include <asm/facility.h> #include <asm/boot_data.h> -#include <asm/switch_to.h> #include "entry.h" #define decompressor_handled_param(param) \ diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S index 49a11f6dd7ae..fc5277eab554 100644 --- a/arch/s390/kernel/entry.S +++ b/arch/s390/kernel/entry.S @@ -24,7 +24,7 @@ #include <asm/page.h> #include <asm/sigp.h> #include <asm/irq.h> -#include <asm/vx-insn.h> +#include <asm/fpu-insn.h> #include <asm/setup.h> #include <asm/nmi.h> #include <asm/nospec-insn.h> @@ -171,13 +171,13 @@ _LPP_OFFSET = __LC_LPP nop 0 /* - * Scheduler resume function, called by switch_to - * gpr2 = (task_struct *) prev - * gpr3 = (task_struct *) next + * Scheduler resume function, called by __switch_to + * gpr2 = (task_struct *)prev + * gpr3 = (task_struct *)next * Returns: * gpr2 = prev */ -SYM_FUNC_START(__switch_to) +SYM_FUNC_START(__switch_to_asm) stmg %r6,%r15,__SF_GPRS(%r15) # store gprs of prev task lghi %r4,__TASK_stack lghi %r1,__TASK_thread @@ -193,7 +193,7 @@ SYM_FUNC_START(__switch_to) lmg %r6,%r15,__SF_GPRS(%r15) # load gprs of next task ALTERNATIVE "nop", "lpp _LPP_OFFSET", 40 BR_EX %r14 -SYM_FUNC_END(__switch_to) +SYM_FUNC_END(__switch_to_asm) #if IS_ENABLED(CONFIG_KVM) /* @@ -220,8 +220,6 @@ SYM_FUNC_START(__sie64a) oi __SIE_PROG0C+3(%r14),1 # we are going into SIE now tm __SIE_PROG20+3(%r14),3 # last exit... jnz .Lsie_skip - TSTMSK __LC_CPU_FLAGS,_CIF_FPU - jo .Lsie_skip # exit if fp/vx regs changed lg %r14,__SF_SIE_CONTROL_PHYS(%r15) # get sie block phys addr BPEXIT __SF_SIE_FLAGS(%r15),_TIF_ISOLATE_BP_GUEST .Lsie_entry: @@ -489,16 +487,11 @@ SYM_FUNC_END(psw_idle) */ SYM_CODE_START(mcck_int_handler) BPOFF - la %r1,4095 # validate r1 - spt __LC_CPU_TIMER_SAVE_AREA-4095(%r1) # validate cpu timer - LBEAR __LC_LAST_BREAK_SAVE_AREA-4095(%r1) # validate bear - lmg %r0,%r15,__LC_GPREGS_SAVE_AREA # validate gprs lmg %r8,%r9,__LC_MCK_OLD_PSW TSTMSK __LC_MCCK_CODE,MCCK_CODE_SYSTEM_DAMAGE jo .Lmcck_panic # yes -> rest of mcck code invalid TSTMSK __LC_MCCK_CODE,MCCK_CODE_CR_VALID jno .Lmcck_panic # control registers invalid -> panic - lctlg %c0,%c15,__LC_CREGS_SAVE_AREA # validate ctl regs ptlb lghi %r14,__LC_CPU_TIMER_SAVE_AREA mvc __LC_MCCK_ENTER_TIMER(8),0(%r14) diff --git a/arch/s390/kernel/entry.h b/arch/s390/kernel/entry.h index 9f41853f36b9..21969520f947 100644 --- a/arch/s390/kernel/entry.h +++ b/arch/s390/kernel/entry.h @@ -19,6 +19,7 @@ void mcck_int_handler(void); void restart_int_handler(void); void early_pgm_check_handler(void); +struct task_struct *__switch_to_asm(struct task_struct *prev, struct task_struct *next); void __ret_from_fork(struct task_struct *prev, struct pt_regs *regs); void __do_pgm_check(struct pt_regs *regs); void __do_syscall(struct pt_regs *regs, int per_trap); diff --git a/arch/s390/kernel/fpu.c b/arch/s390/kernel/fpu.c index a4f3449cc814..fa90bbdc5ef9 100644 --- a/arch/s390/kernel/fpu.c +++ b/arch/s390/kernel/fpu.c @@ -8,256 +8,186 @@ #include <linux/kernel.h> #include <linux/cpu.h> #include <linux/sched.h> -#include <asm/fpu/types.h> -#include <asm/fpu/api.h> -#include <asm/vx-insn.h> +#include <asm/fpu.h> -void __kernel_fpu_begin(struct kernel_fpu *state, u32 flags) +void __kernel_fpu_begin(struct kernel_fpu *state, int flags) { + __vector128 *vxrs = state->vxrs; + int mask; + /* * Limit the save to the FPU/vector registers already - * in use by the previous context + * in use by the previous context. */ - flags &= state->mask; - + flags &= state->hdr.mask; if (flags & KERNEL_FPC) - /* Save floating point control */ - asm volatile("stfpc %0" : "=Q" (state->fpc)); - + fpu_stfpc(&state->hdr.fpc); if (!cpu_has_vx()) { - if (flags & KERNEL_VXR_V0V7) { - /* Save floating-point registers */ - asm volatile("std 0,%0" : "=Q" (state->fprs[0])); - asm volatile("std 1,%0" : "=Q" (state->fprs[1])); - asm volatile("std 2,%0" : "=Q" (state->fprs[2])); - asm volatile("std 3,%0" : "=Q" (state->fprs[3])); - asm volatile("std 4,%0" : "=Q" (state->fprs[4])); - asm volatile("std 5,%0" : "=Q" (state->fprs[5])); - asm volatile("std 6,%0" : "=Q" (state->fprs[6])); - asm volatile("std 7,%0" : "=Q" (state->fprs[7])); - asm volatile("std 8,%0" : "=Q" (state->fprs[8])); - asm volatile("std 9,%0" : "=Q" (state->fprs[9])); - asm volatile("std 10,%0" : "=Q" (state->fprs[10])); - asm volatile("std 11,%0" : "=Q" (state->fprs[11])); - asm volatile("std 12,%0" : "=Q" (state->fprs[12])); - asm volatile("std 13,%0" : "=Q" (state->fprs[13])); - asm volatile("std 14,%0" : "=Q" (state->fprs[14])); - asm volatile("std 15,%0" : "=Q" (state->fprs[15])); - } + if (flags & KERNEL_VXR_LOW) + save_fp_regs_vx(vxrs); return; } - - /* Test and save vector registers */ - asm volatile ( - /* - * Test if any vector register must be saved and, if so, - * test if all register can be saved. - */ - " la 1,%[vxrs]\n" /* load save area */ - " tmll %[m],30\n" /* KERNEL_VXR */ - " jz 7f\n" /* no work -> done */ - " jo 5f\n" /* -> save V0..V31 */ - /* - * Test for special case KERNEL_FPU_MID only. In this - * case a vstm V8..V23 is the best instruction - */ - " chi %[m],12\n" /* KERNEL_VXR_MID */ - " jne 0f\n" /* -> save V8..V23 */ - " VSTM 8,23,128,1\n" /* vstm %v8,%v23,128(%r1) */ - " j 7f\n" - /* Test and save the first half of 16 vector registers */ - "0: tmll %[m],6\n" /* KERNEL_VXR_LOW */ - " jz 3f\n" /* -> KERNEL_VXR_HIGH */ - " jo 2f\n" /* 11 -> save V0..V15 */ - " brc 2,1f\n" /* 10 -> save V8..V15 */ - " VSTM 0,7,0,1\n" /* vstm %v0,%v7,0(%r1) */ - " j 3f\n" - "1: VSTM 8,15,128,1\n" /* vstm %v8,%v15,128(%r1) */ - " j 3f\n" - "2: VSTM 0,15,0,1\n" /* vstm %v0,%v15,0(%r1) */ - /* Test and save the second half of 16 vector registers */ - "3: tmll %[m],24\n" /* KERNEL_VXR_HIGH */ - " jz 7f\n" - " jo 6f\n" /* 11 -> save V16..V31 */ - " brc 2,4f\n" /* 10 -> save V24..V31 */ - " VSTM 16,23,256,1\n" /* vstm %v16,%v23,256(%r1) */ - " j 7f\n" - "4: VSTM 24,31,384,1\n" /* vstm %v24,%v31,384(%r1) */ - " j 7f\n" - "5: VSTM 0,15,0,1\n" /* vstm %v0,%v15,0(%r1) */ - "6: VSTM 16,31,256,1\n" /* vstm %v16,%v31,256(%r1) */ - "7:" - : [vxrs] "=Q" (*(struct vx_array *) &state->vxrs) - : [m] "d" (flags) - : "1", "cc"); + mask = flags & KERNEL_VXR; + if (mask == KERNEL_VXR) { + vxrs += fpu_vstm(0, 15, vxrs); + vxrs += fpu_vstm(16, 31, vxrs); + return; + } + if (mask == KERNEL_VXR_MID) { + vxrs += fpu_vstm(8, 23, vxrs); + return; + } + mask = flags & KERNEL_VXR_LOW; + if (mask) { + if (mask == KERNEL_VXR_LOW) + vxrs += fpu_vstm(0, 15, vxrs); + else if (mask == KERNEL_VXR_V0V7) + vxrs += fpu_vstm(0, 7, vxrs); + else + vxrs += fpu_vstm(8, 15, vxrs); + } + mask = flags & KERNEL_VXR_HIGH; + if (mask) { + if (mask == KERNEL_VXR_HIGH) + vxrs += fpu_vstm(16, 31, vxrs); + else if (mask == KERNEL_VXR_V16V23) + vxrs += fpu_vstm(16, 23, vxrs); + else + vxrs += fpu_vstm(24, 31, vxrs); + } } EXPORT_SYMBOL(__kernel_fpu_begin); -void __kernel_fpu_end(struct kernel_fpu *state, u32 flags) +void __kernel_fpu_end(struct kernel_fpu *state, int flags) { + __vector128 *vxrs = state->vxrs; + int mask; + /* * Limit the restore to the FPU/vector registers of the - * previous context that have been overwritte by the - * current context + * previous context that have been overwritten by the + * current context. */ - flags &= state->mask; - + flags &= state->hdr.mask; if (flags & KERNEL_FPC) - /* Restore floating-point controls */ - asm volatile("lfpc %0" : : "Q" (state->fpc)); - + fpu_lfpc(&state->hdr.fpc); if (!cpu_has_vx()) { - if (flags & KERNEL_VXR_V0V7) { - /* Restore floating-point registers */ - asm volatile("ld 0,%0" : : "Q" (state->fprs[0])); - asm volatile("ld 1,%0" : : "Q" (state->fprs[1])); - asm volatile("ld 2,%0" : : "Q" (state->fprs[2])); - asm volatile("ld 3,%0" : : "Q" (state->fprs[3])); - asm volatile("ld 4,%0" : : "Q" (state->fprs[4])); - asm volatile("ld 5,%0" : : "Q" (state->fprs[5])); - asm volatile("ld 6,%0" : : "Q" (state->fprs[6])); - asm volatile("ld 7,%0" : : "Q" (state->fprs[7])); - asm volatile("ld 8,%0" : : "Q" (state->fprs[8])); - asm volatile("ld 9,%0" : : "Q" (state->fprs[9])); - asm volatile("ld 10,%0" : : "Q" (state->fprs[10])); - asm volatile("ld 11,%0" : : "Q" (state->fprs[11])); - asm volatile("ld 12,%0" : : "Q" (state->fprs[12])); - asm volatile("ld 13,%0" : : "Q" (state->fprs[13])); - asm volatile("ld 14,%0" : : "Q" (state->fprs[14])); - asm volatile("ld 15,%0" : : "Q" (state->fprs[15])); - } + if (flags & KERNEL_VXR_LOW) + load_fp_regs_vx(vxrs); return; } - - /* Test and restore (load) vector registers */ - asm volatile ( - /* - * Test if any vector register must be loaded and, if so, - * test if all registers can be loaded at once. - */ - " la 1,%[vxrs]\n" /* load restore area */ - " tmll %[m],30\n" /* KERNEL_VXR */ - " jz 7f\n" /* no work -> done */ - " jo 5f\n" /* -> restore V0..V31 */ - /* - * Test for special case KERNEL_FPU_MID only. In this - * case a vlm V8..V23 is the best instruction - */ - " chi %[m],12\n" /* KERNEL_VXR_MID */ - " jne 0f\n" /* -> restore V8..V23 */ - " VLM 8,23,128,1\n" /* vlm %v8,%v23,128(%r1) */ - " j 7f\n" - /* Test and restore the first half of 16 vector registers */ - "0: tmll %[m],6\n" /* KERNEL_VXR_LOW */ - " jz 3f\n" /* -> KERNEL_VXR_HIGH */ - " jo 2f\n" /* 11 -> restore V0..V15 */ - " brc 2,1f\n" /* 10 -> restore V8..V15 */ - " VLM 0,7,0,1\n" /* vlm %v0,%v7,0(%r1) */ - " j 3f\n" - "1: VLM 8,15,128,1\n" /* vlm %v8,%v15,128(%r1) */ - " j 3f\n" - "2: VLM 0,15,0,1\n" /* vlm %v0,%v15,0(%r1) */ - /* Test and restore the second half of 16 vector registers */ - "3: tmll %[m],24\n" /* KERNEL_VXR_HIGH */ - " jz 7f\n" - " jo 6f\n" /* 11 -> restore V16..V31 */ - " brc 2,4f\n" /* 10 -> restore V24..V31 */ - " VLM 16,23,256,1\n" /* vlm %v16,%v23,256(%r1) */ - " j 7f\n" - "4: VLM 24,31,384,1\n" /* vlm %v24,%v31,384(%r1) */ - " j 7f\n" - "5: VLM 0,15,0,1\n" /* vlm %v0,%v15,0(%r1) */ - "6: VLM 16,31,256,1\n" /* vlm %v16,%v31,256(%r1) */ - "7:" - : [vxrs] "=Q" (*(struct vx_array *) &state->vxrs) - : [m] "d" (flags) - : "1", "cc"); + mask = flags & KERNEL_VXR; + if (mask == KERNEL_VXR) { + vxrs += fpu_vlm(0, 15, vxrs); + vxrs += fpu_vlm(16, 31, vxrs); + return; + } + if (mask == KERNEL_VXR_MID) { + vxrs += fpu_vlm(8, 23, vxrs); + return; + } + mask = flags & KERNEL_VXR_LOW; + if (mask) { + if (mask == KERNEL_VXR_LOW) + vxrs += fpu_vlm(0, 15, vxrs); + else if (mask == KERNEL_VXR_V0V7) + vxrs += fpu_vlm(0, 7, vxrs); + else + vxrs += fpu_vlm(8, 15, vxrs); + } + mask = flags & KERNEL_VXR_HIGH; + if (mask) { + if (mask == KERNEL_VXR_HIGH) + vxrs += fpu_vlm(16, 31, vxrs); + else if (mask == KERNEL_VXR_V16V23) + vxrs += fpu_vlm(16, 23, vxrs); + else + vxrs += fpu_vlm(24, 31, vxrs); + } } EXPORT_SYMBOL(__kernel_fpu_end); -void __load_fpu_regs(void) +void load_fpu_state(struct fpu *state, int flags) { - unsigned long *regs = current->thread.fpu.regs; - struct fpu *state = ¤t->thread.fpu; + __vector128 *vxrs = &state->vxrs[0]; + int mask; - sfpc_safe(state->fpc); - if (likely(cpu_has_vx())) { - asm volatile("lgr 1,%0\n" - "VLM 0,15,0,1\n" - "VLM 16,31,256,1\n" - : - : "d" (regs) - : "1", "cc", "memory"); - } else { - asm volatile("ld 0,%0" : : "Q" (regs[0])); - asm volatile("ld 1,%0" : : "Q" (regs[1])); - asm volatile("ld 2,%0" : : "Q" (regs[2])); - asm volatile("ld 3,%0" : : "Q" (regs[3])); - asm volatile("ld 4,%0" : : "Q" (regs[4])); - asm volatile("ld 5,%0" : : "Q" (regs[5])); - asm volatile("ld 6,%0" : : "Q" (regs[6])); - asm volatile("ld 7,%0" : : "Q" (regs[7])); - asm volatile("ld 8,%0" : : "Q" (regs[8])); - asm volatile("ld 9,%0" : : "Q" (regs[9])); - asm volatile("ld 10,%0" : : "Q" (regs[10])); - asm volatile("ld 11,%0" : : "Q" (regs[11])); - asm volatile("ld 12,%0" : : "Q" (regs[12])); - asm volatile("ld 13,%0" : : "Q" (regs[13])); - asm volatile("ld 14,%0" : : "Q" (regs[14])); - asm volatile("ld 15,%0" : : "Q" (regs[15])); + if (flags & KERNEL_FPC) + fpu_lfpc(&state->fpc); + if (!cpu_has_vx()) { + if (flags & KERNEL_VXR_V0V7) + load_fp_regs_vx(state->vxrs); + return; + } + mask = flags & KERNEL_VXR; + if (mask == KERNEL_VXR) { + fpu_vlm(0, 15, &vxrs[0]); + fpu_vlm(16, 31, &vxrs[16]); + return; + } + if (mask == KERNEL_VXR_MID) { + fpu_vlm(8, 23, &vxrs[8]); + return; + } + mask = flags & KERNEL_VXR_LOW; + if (mask) { + if (mask == KERNEL_VXR_LOW) + fpu_vlm(0, 15, &vxrs[0]); + else if (mask == KERNEL_VXR_V0V7) + fpu_vlm(0, 7, &vxrs[0]); + else + fpu_vlm(8, 15, &vxrs[8]); + } + mask = flags & KERNEL_VXR_HIGH; + if (mask) { + if (mask == KERNEL_VXR_HIGH) + fpu_vlm(16, 31, &vxrs[16]); + else if (mask == KERNEL_VXR_V16V23) + fpu_vlm(16, 23, &vxrs[16]); + else + fpu_vlm(24, 31, &vxrs[24]); } - clear_cpu_flag(CIF_FPU); -} - -void load_fpu_regs(void) -{ - raw_local_irq_disable(); - __load_fpu_regs(); - raw_local_irq_enable(); } -EXPORT_SYMBOL(load_fpu_regs); -void save_fpu_regs(void) +void save_fpu_state(struct fpu *state, int flags) { - unsigned long flags, *regs; - struct fpu *state; - - local_irq_save(flags); + __vector128 *vxrs = &state->vxrs[0]; + int mask; - if (test_cpu_flag(CIF_FPU)) - goto out; - - state = ¤t->thread.fpu; - regs = current->thread.fpu.regs; - - asm volatile("stfpc %0" : "=Q" (state->fpc)); - if (likely(cpu_has_vx())) { - asm volatile("lgr 1,%0\n" - "VSTM 0,15,0,1\n" - "VSTM 16,31,256,1\n" - : - : "d" (regs) - : "1", "cc", "memory"); - } else { - asm volatile("std 0,%0" : "=Q" (regs[0])); - asm volatile("std 1,%0" : "=Q" (regs[1])); - asm volatile("std 2,%0" : "=Q" (regs[2])); - asm volatile("std 3,%0" : "=Q" (regs[3])); - asm volatile("std 4,%0" : "=Q" (regs[4])); - asm volatile("std 5,%0" : "=Q" (regs[5])); - asm volatile("std 6,%0" : "=Q" (regs[6])); - asm volatile("std 7,%0" : "=Q" (regs[7])); - asm volatile("std 8,%0" : "=Q" (regs[8])); - asm volatile("std 9,%0" : "=Q" (regs[9])); - asm volatile("std 10,%0" : "=Q" (regs[10])); - asm volatile("std 11,%0" : "=Q" (regs[11])); - asm volatile("std 12,%0" : "=Q" (regs[12])); - asm volatile("std 13,%0" : "=Q" (regs[13])); - asm volatile("std 14,%0" : "=Q" (regs[14])); - asm volatile("std 15,%0" : "=Q" (regs[15])); + if (flags & KERNEL_FPC) + fpu_stfpc(&state->fpc); + if (!cpu_has_vx()) { + if (flags & KERNEL_VXR_LOW) + save_fp_regs_vx(state->vxrs); + return; + } + mask = flags & KERNEL_VXR; + if (mask == KERNEL_VXR) { + fpu_vstm(0, 15, &vxrs[0]); + fpu_vstm(16, 31, &vxrs[16]); + return; + } + if (mask == KERNEL_VXR_MID) { + fpu_vstm(8, 23, &vxrs[8]); + return; + } + mask = flags & KERNEL_VXR_LOW; + if (mask) { + if (mask == KERNEL_VXR_LOW) + fpu_vstm(0, 15, &vxrs[0]); + else if (mask == KERNEL_VXR_V0V7) + fpu_vstm(0, 7, &vxrs[0]); + else + fpu_vstm(8, 15, &vxrs[8]); + } + mask = flags & KERNEL_VXR_HIGH; + if (mask) { + if (mask == KERNEL_VXR_HIGH) + fpu_vstm(16, 31, &vxrs[16]); + else if (mask == KERNEL_VXR_V16V23) + fpu_vstm(16, 23, &vxrs[16]); + else + fpu_vstm(24, 31, &vxrs[24]); } - set_cpu_flag(CIF_FPU); -out: - local_irq_restore(flags); } -EXPORT_SYMBOL(save_fpu_regs); +EXPORT_SYMBOL(save_fpu_state); diff --git a/arch/s390/kernel/ipl.c b/arch/s390/kernel/ipl.c index ba75f6bee774..1486350a4177 100644 --- a/arch/s390/kernel/ipl.c +++ b/arch/s390/kernel/ipl.c @@ -1941,8 +1941,7 @@ static void dump_reipl_run(struct shutdown_trigger *trigger) reipl_type == IPL_TYPE_UNKNOWN) os_info_flags |= OS_INFO_FLAG_REIPL_CLEAR; os_info_entry_add(OS_INFO_FLAGS_ENTRY, &os_info_flags, sizeof(os_info_flags)); - csum = (__force unsigned int) - csum_partial(reipl_block_actual, reipl_block_actual->hdr.len, 0); + csum = (__force unsigned int)cksm(reipl_block_actual, reipl_block_actual->hdr.len, 0); abs_lc = get_abs_lowcore(); abs_lc->ipib = __pa(reipl_block_actual); abs_lc->ipib_checksum = csum; diff --git a/arch/s390/kernel/machine_kexec.c b/arch/s390/kernel/machine_kexec.c index aa22ffc16bcd..c5d0c1cf984b 100644 --- a/arch/s390/kernel/machine_kexec.c +++ b/arch/s390/kernel/machine_kexec.c @@ -13,8 +13,10 @@ #include <linux/reboot.h> #include <linux/ftrace.h> #include <linux/debug_locks.h> +#include <asm/guarded_storage.h> #include <asm/pfault.h> #include <asm/cio.h> +#include <asm/fpu.h> #include <asm/setup.h> #include <asm/smp.h> #include <asm/ipl.h> @@ -26,7 +28,6 @@ #include <asm/os_info.h> #include <asm/set_memory.h> #include <asm/stacktrace.h> -#include <asm/switch_to.h> #include <asm/nmi.h> #include <asm/sclp.h> diff --git a/arch/s390/kernel/nmi.c b/arch/s390/kernel/nmi.c index 9ad44c26d1a2..c77382a67325 100644 --- a/arch/s390/kernel/nmi.c +++ b/arch/s390/kernel/nmi.c @@ -23,16 +23,14 @@ #include <linux/export.h> #include <asm/lowcore.h> #include <asm/ctlreg.h> +#include <asm/fpu.h> #include <asm/smp.h> #include <asm/stp.h> #include <asm/cputime.h> #include <asm/nmi.h> #include <asm/crw.h> -#include <asm/switch_to.h> #include <asm/asm-offsets.h> #include <asm/pai.h> -#include <asm/vx-insn.h> -#include <asm/fpu/api.h> struct mcck_struct { unsigned int kill_task : 1; @@ -204,133 +202,63 @@ void s390_handle_mcck(void) } } -/* - * returns 0 if register contents could be validated - * returns 1 otherwise +/** + * nmi_registers_valid - verify if registers are valid + * @mci: machine check interruption code + * + * Inspect a machine check interruption code and verify if all required + * registers are valid. For some registers the corresponding validity bit is + * ignored and the registers are set to the expected value. + * Returns true if all registers are valid, otherwise false. */ -static int notrace s390_validate_registers(union mci mci) +static bool notrace nmi_registers_valid(union mci mci) { - struct mcesa *mcesa; - void *fpt_save_area; union ctlreg2 cr2; - int kill_task; - u64 zero; - - kill_task = 0; - zero = 0; - - if (!mci.gr || !mci.fp) - kill_task = 1; - fpt_save_area = &S390_lowcore.floating_pt_save_area; - if (!mci.fc) { - kill_task = 1; - asm volatile( - " lfpc %0\n" - : - : "Q" (zero)); - } else { - asm volatile( - " lfpc %0\n" - : - : "Q" (S390_lowcore.fpt_creg_save_area)); - } - mcesa = __va(S390_lowcore.mcesad & MCESA_ORIGIN_MASK); - if (!cpu_has_vx()) { - /* Validate floating point registers */ - asm volatile( - " ld 0,0(%0)\n" - " ld 1,8(%0)\n" - " ld 2,16(%0)\n" - " ld 3,24(%0)\n" - " ld 4,32(%0)\n" - " ld 5,40(%0)\n" - " ld 6,48(%0)\n" - " ld 7,56(%0)\n" - " ld 8,64(%0)\n" - " ld 9,72(%0)\n" - " ld 10,80(%0)\n" - " ld 11,88(%0)\n" - " ld 12,96(%0)\n" - " ld 13,104(%0)\n" - " ld 14,112(%0)\n" - " ld 15,120(%0)\n" - : - : "a" (fpt_save_area) - : "memory"); - } else { - /* Validate vector registers */ - union ctlreg0 cr0; - - /* - * The vector validity must only be checked if not running a - * KVM guest. For KVM guests the machine check is forwarded by - * KVM and it is the responsibility of the guest to take - * appropriate actions. The host vector or FPU values have been - * saved by KVM and will be restored by KVM. - */ - if (!mci.vr && !test_cpu_flag(CIF_MCCK_GUEST)) - kill_task = 1; - cr0.reg = S390_lowcore.cregs_save_area[0]; - cr0.afp = cr0.vx = 1; - local_ctl_load(0, &cr0.reg); - asm volatile( - " la 1,%0\n" - " VLM 0,15,0,1\n" - " VLM 16,31,256,1\n" - : - : "Q" (*(struct vx_array *)mcesa->vector_save_area) - : "1"); - local_ctl_load(0, &S390_lowcore.cregs_save_area[0]); - } - /* Validate access registers */ - asm volatile( - " lam 0,15,0(%0)\n" - : - : "a" (&S390_lowcore.access_regs_save_area) - : "memory"); - if (!mci.ar) - kill_task = 1; - /* Validate guarded storage registers */ - cr2.reg = S390_lowcore.cregs_save_area[2]; - if (cr2.gse) { - if (!mci.gs) { - /* - * 2 cases: - * - machine check in kernel or userspace - * - machine check while running SIE (KVM guest) - * For kernel or userspace the userspace values of - * guarded storage control can not be recreated, the - * process must be terminated. - * For SIE the guest values of guarded storage can not - * be recreated. This is either due to a bug or due to - * GS being disabled in the guest. The guest will be - * notified by KVM code and the guests machine check - * handling must take care of this. The host values - * are saved by KVM and are not affected. - */ - if (!test_cpu_flag(CIF_MCCK_GUEST)) - kill_task = 1; - } else { - load_gs_cb((struct gs_cb *)mcesa->guarded_storage_save_area); - } - } /* - * The getcpu vdso syscall reads CPU number from the programmable + * The getcpu vdso syscall reads the CPU number from the programmable * field of the TOD clock. Disregard the TOD programmable register - * validity bit and load the CPU number into the TOD programmable - * field unconditionally. + * validity bit and load the CPU number into the TOD programmable field + * unconditionally. */ set_tod_programmable_field(raw_smp_processor_id()); - /* Validate clock comparator register */ + /* + * Set the clock comparator register to the next expected value. + */ set_clock_comparator(S390_lowcore.clock_comparator); - + if (!mci.gr || !mci.fp || !mci.fc) + return false; + /* + * The vector validity must only be checked if not running a + * KVM guest. For KVM guests the machine check is forwarded by + * KVM and it is the responsibility of the guest to take + * appropriate actions. The host vector or FPU values have been + * saved by KVM and will be restored by KVM. + */ + if (!mci.vr && !test_cpu_flag(CIF_MCCK_GUEST)) + return false; + if (!mci.ar) + return false; + /* + * Two cases for guarded storage registers: + * - machine check in kernel or userspace + * - machine check while running SIE (KVM guest) + * For kernel or userspace the userspace values of guarded storage + * control can not be recreated, the process must be terminated. + * For SIE the guest values of guarded storage can not be recreated. + * This is either due to a bug or due to GS being disabled in the + * guest. The guest will be notified by KVM code and the guests machine + * check handling must take care of this. The host values are saved by + * KVM and are not affected. + */ + cr2.reg = S390_lowcore.cregs_save_area[2]; + if (cr2.gse && !mci.gs && !test_cpu_flag(CIF_MCCK_GUEST)) + return false; if (!mci.ms || !mci.pm || !mci.ia) - kill_task = 1; - - return kill_task; + return false; + return true; } -NOKPROBE_SYMBOL(s390_validate_registers); +NOKPROBE_SYMBOL(nmi_registers_valid); /* * Backup the guest's machine check info to its description block @@ -428,7 +356,7 @@ void notrace s390_do_machine_check(struct pt_regs *regs) s390_handle_damage(); } } - if (s390_validate_registers(mci)) { + if (!nmi_registers_valid(mci)) { if (!user_mode(regs)) s390_handle_damage(); /* diff --git a/arch/s390/kernel/os_info.c b/arch/s390/kernel/os_info.c index 6e1824141b29..a801e6bd5341 100644 --- a/arch/s390/kernel/os_info.c +++ b/arch/s390/kernel/os_info.c @@ -29,7 +29,7 @@ static struct os_info os_info __page_aligned_data; u32 os_info_csum(struct os_info *os_info) { int size = sizeof(*os_info) - offsetof(struct os_info, version_major); - return (__force u32)csum_partial(&os_info->version_major, size, 0); + return (__force u32)cksm(&os_info->version_major, size, 0); } /* @@ -49,7 +49,7 @@ void os_info_entry_add(int nr, void *ptr, u64 size) { os_info.entry[nr].addr = __pa(ptr); os_info.entry[nr].size = size; - os_info.entry[nr].csum = (__force u32)csum_partial(ptr, size, 0); + os_info.entry[nr].csum = (__force u32)cksm(ptr, size, 0); os_info.csum = os_info_csum(&os_info); } @@ -98,7 +98,7 @@ static void os_info_old_alloc(int nr, int align) msg = "copy failed"; goto fail_free; } - csum = (__force u32)csum_partial(buf_align, size, 0); + csum = (__force u32)cksm(buf_align, size, 0); if (csum != os_info_old->entry[nr].csum) { msg = "checksum failed"; goto fail_free; diff --git a/arch/s390/kernel/perf_pai_crypto.c b/arch/s390/kernel/perf_pai_crypto.c index bf8a672b15a4..823d652e3917 100644 --- a/arch/s390/kernel/perf_pai_crypto.c +++ b/arch/s390/kernel/perf_pai_crypto.c @@ -98,6 +98,7 @@ static void paicrypt_event_destroy(struct perf_event *event) event->attr.config, event->cpu, cpump->active_events, cpump->mode, refcount_read(&cpump->refcnt)); + free_page(PAI_SAVE_AREA(event)); if (refcount_dec_and_test(&cpump->refcnt)) { debug_sprintf_event(cfm_dbg, 4, "%s page %#lx save %p\n", __func__, (unsigned long)cpump->page, @@ -260,6 +261,7 @@ static int paicrypt_event_init(struct perf_event *event) { struct perf_event_attr *a = &event->attr; struct paicrypt_map *cpump; + int rc = 0; /* PAI crypto PMU registered as PERF_TYPE_RAW, check event type */ if (a->type != PERF_TYPE_RAW && event->pmu->type != a->type) @@ -274,10 +276,21 @@ static int paicrypt_event_init(struct perf_event *event) /* Allow only CRYPTO_ALL for sampling. */ if (a->sample_period && a->config != PAI_CRYPTO_BASE) return -EINVAL; + /* Get a page to store last counter values for sampling */ + if (a->sample_period) { + PAI_SAVE_AREA(event) = get_zeroed_page(GFP_KERNEL); + if (!PAI_SAVE_AREA(event)) { + rc = -ENOMEM; + goto out; + } + } cpump = paicrypt_busy(event); - if (IS_ERR(cpump)) - return PTR_ERR(cpump); + if (IS_ERR(cpump)) { + free_page(PAI_SAVE_AREA(event)); + rc = PTR_ERR(cpump); + goto out; + } event->destroy = paicrypt_event_destroy; @@ -293,7 +306,8 @@ static int paicrypt_event_init(struct perf_event *event) } static_branch_inc(&pai_key); - return 0; +out: + return rc; } static void paicrypt_read(struct perf_event *event) @@ -310,20 +324,15 @@ static void paicrypt_read(struct perf_event *event) static void paicrypt_start(struct perf_event *event, int flags) { + struct paicrypt_mapptr *mp = this_cpu_ptr(paicrypt_root.mapptr); + struct paicrypt_map *cpump = mp->mapptr; u64 sum; - /* Event initialization sets last_tag to 0. When later on the events - * are deleted and re-added, do not reset the event count value to zero. - * Events are added, deleted and re-added when 2 or more events - * are active at the same time. - */ if (!event->attr.sample_period) { /* Counting */ - if (!event->hw.last_tag) { - event->hw.last_tag = 1; - sum = paicrypt_getall(event); /* Get current value */ - local64_set(&event->hw.prev_count, sum); - } + sum = paicrypt_getall(event); /* Get current value */ + local64_set(&event->hw.prev_count, sum); } else { /* Sampling */ + cpump->event = event; perf_sched_cb_inc(event->pmu); } } @@ -339,7 +348,6 @@ static int paicrypt_add(struct perf_event *event, int flags) WRITE_ONCE(S390_lowcore.ccd, ccd); local_ctl_set_bit(0, CR0_CRYPTOGRAPHY_COUNTER_BIT); } - cpump->event = event; if (flags & PERF_EF_START) paicrypt_start(event, PERF_EF_RELOAD); event->hw.state = 0; @@ -367,23 +375,34 @@ static void paicrypt_del(struct perf_event *event, int flags) } } -/* Create raw data and save it in buffer. Returns number of bytes copied. - * Saves only positive counter entries of the form +/* Create raw data and save it in buffer. Calculate the delta for each + * counter between this invocation and the last invocation. + * Returns number of bytes copied. + * Saves only entries with positive counter difference of the form * 2 bytes: Number of counter * 8 bytes: Value of counter */ static size_t paicrypt_copy(struct pai_userdata *userdata, unsigned long *page, - bool exclude_user, bool exclude_kernel) + unsigned long *page_old, bool exclude_user, + bool exclude_kernel) { int i, outidx = 0; for (i = 1; i <= paicrypt_cnt; i++) { - u64 val = 0; + u64 val = 0, val_old = 0; - if (!exclude_kernel) + if (!exclude_kernel) { val += paicrypt_getctr(page, i, true); - if (!exclude_user) + val_old += paicrypt_getctr(page_old, i, true); + } + if (!exclude_user) { val += paicrypt_getctr(page, i, false); + val_old += paicrypt_getctr(page_old, i, false); + } + if (val >= val_old) + val -= val_old; + else + val = (~0ULL - val_old) + val + 1; if (val) { userdata[outidx].num = i; userdata[outidx].value = val; @@ -426,8 +445,8 @@ static int paicrypt_push_sample(size_t rawsize, struct paicrypt_map *cpump, overflow = perf_event_overflow(event, &data, ®s); perf_event_update_userpage(event); - /* Clear lowcore page after read */ - memset(cpump->page, 0, PAGE_SIZE); + /* Save crypto counter lowcore page after reading event data. */ + memcpy((void *)PAI_SAVE_AREA(event), cpump->page, PAGE_SIZE); return overflow; } @@ -443,6 +462,7 @@ static int paicrypt_have_sample(void) if (!event) /* No event active */ return 0; rawsize = paicrypt_copy(cpump->save, cpump->page, + (unsigned long *)PAI_SAVE_AREA(event), cpump->event->attr.exclude_user, cpump->event->attr.exclude_kernel); if (rawsize) /* No incremented counters */ @@ -694,6 +714,12 @@ static int __init attr_event_init_one(struct attribute **attrs, int num) { struct perf_pmu_events_attr *pa; + /* Index larger than array_size, no counter name available */ + if (num >= ARRAY_SIZE(paicrypt_ctrnames)) { + attrs[num] = NULL; + return 0; + } + pa = kzalloc(sizeof(*pa), GFP_KERNEL); if (!pa) return -ENOMEM; @@ -714,14 +740,13 @@ static int __init attr_event_init(void) struct attribute **attrs; int ret, i; - attrs = kmalloc_array(ARRAY_SIZE(paicrypt_ctrnames) + 1, sizeof(*attrs), - GFP_KERNEL); + attrs = kmalloc_array(paicrypt_cnt + 2, sizeof(*attrs), GFP_KERNEL); if (!attrs) return -ENOMEM; - for (i = 0; i < ARRAY_SIZE(paicrypt_ctrnames); i++) { + for (i = 0; i <= paicrypt_cnt; i++) { ret = attr_event_init_one(attrs, i); if (ret) { - attr_event_free(attrs, i - 1); + attr_event_free(attrs, i); return ret; } } @@ -742,8 +767,10 @@ static int __init paicrypt_init(void) paicrypt_cnt = ib.num_cc; if (paicrypt_cnt == 0) return 0; - if (paicrypt_cnt >= PAI_CRYPTO_MAXCTR) - paicrypt_cnt = PAI_CRYPTO_MAXCTR - 1; + if (paicrypt_cnt >= PAI_CRYPTO_MAXCTR) { + pr_err("Too many PMU pai_crypto counters %d\n", paicrypt_cnt); + return -E2BIG; + } rc = attr_event_init(); /* Export known PAI crypto events */ if (rc) { diff --git a/arch/s390/kernel/perf_pai_ext.c b/arch/s390/kernel/perf_pai_ext.c index af7f2b538c8f..616a25606cd6 100644 --- a/arch/s390/kernel/perf_pai_ext.c +++ b/arch/s390/kernel/perf_pai_ext.c @@ -120,6 +120,7 @@ static void paiext_event_destroy(struct perf_event *event) struct paiext_mapptr *mp = per_cpu_ptr(paiext_root.mapptr, event->cpu); struct paiext_map *cpump = mp->mapptr; + free_page(PAI_SAVE_AREA(event)); mutex_lock(&paiext_reserve_mutex); cpump->event = NULL; if (refcount_dec_and_test(&cpump->refcnt)) /* Last reference gone */ @@ -202,7 +203,6 @@ static int paiext_alloc(struct perf_event_attr *a, struct perf_event *event) } rc = 0; - cpump->event = event; undo: if (rc) { @@ -256,10 +256,18 @@ static int paiext_event_init(struct perf_event *event) /* Prohibit exclude_user event selection */ if (a->exclude_user) return -EINVAL; + /* Get a page to store last counter values for sampling */ + if (a->sample_period) { + PAI_SAVE_AREA(event) = get_zeroed_page(GFP_KERNEL); + if (!PAI_SAVE_AREA(event)) + return -ENOMEM; + } rc = paiext_alloc(a, event); - if (rc) + if (rc) { + free_page(PAI_SAVE_AREA(event)); return rc; + } event->destroy = paiext_event_destroy; if (a->sample_period) { @@ -319,15 +327,15 @@ static void paiext_read(struct perf_event *event) static void paiext_start(struct perf_event *event, int flags) { + struct paiext_mapptr *mp = this_cpu_ptr(paiext_root.mapptr); + struct paiext_map *cpump = mp->mapptr; u64 sum; if (!event->attr.sample_period) { /* Counting */ - if (!event->hw.last_tag) { - event->hw.last_tag = 1; - sum = paiext_getall(event); /* Get current value */ - local64_set(&event->hw.prev_count, sum); - } + sum = paiext_getall(event); /* Get current value */ + local64_set(&event->hw.prev_count, sum); } else { /* Sampling */ + cpump->event = event; perf_sched_cb_inc(event->pmu); } } @@ -346,7 +354,6 @@ static int paiext_add(struct perf_event *event, int flags) debug_sprintf_event(paiext_dbg, 4, "%s 1508 %llx acc %llx\n", __func__, S390_lowcore.aicd, pcb->acc); } - cpump->event = event; if (flags & PERF_EF_START) paiext_start(event, PERF_EF_RELOAD); event->hw.state = 0; @@ -384,13 +391,19 @@ static void paiext_del(struct perf_event *event, int flags) * 2 bytes: Number of counter * 8 bytes: Value of counter */ -static size_t paiext_copy(struct pai_userdata *userdata, unsigned long *area) +static size_t paiext_copy(struct pai_userdata *userdata, unsigned long *area, + unsigned long *area_old) { int i, outidx = 0; for (i = 1; i <= paiext_cnt; i++) { u64 val = paiext_getctr(area, i); + u64 val_old = paiext_getctr(area_old, i); + if (val >= val_old) + val -= val_old; + else + val = (~0ULL - val_old) + val + 1; if (val) { userdata[outidx].num = i; userdata[outidx].value = val; @@ -446,8 +459,9 @@ static int paiext_push_sample(size_t rawsize, struct paiext_map *cpump, overflow = perf_event_overflow(event, &data, ®s); perf_event_update_userpage(event); - /* Clear lowcore area after read */ - memset(cpump->area, 0, PAIE1_CTRBLOCK_SZ); + /* Save NNPA lowcore area after read in event */ + memcpy((void *)PAI_SAVE_AREA(event), cpump->area, + PAIE1_CTRBLOCK_SZ); return overflow; } @@ -462,7 +476,8 @@ static int paiext_have_sample(void) if (!event) return 0; - rawsize = paiext_copy(cpump->save, cpump->area); + rawsize = paiext_copy(cpump->save, cpump->area, + (unsigned long *)PAI_SAVE_AREA(event)); if (rawsize) /* Incremented counters */ rc = paiext_push_sample(rawsize, cpump, event); return rc; @@ -584,6 +599,12 @@ static int __init attr_event_init_one(struct attribute **attrs, int num) { struct perf_pmu_events_attr *pa; + /* Index larger than array_size, no counter name available */ + if (num >= ARRAY_SIZE(paiext_ctrnames)) { + attrs[num] = NULL; + return 0; + } + pa = kzalloc(sizeof(*pa), GFP_KERNEL); if (!pa) return -ENOMEM; @@ -604,14 +625,13 @@ static int __init attr_event_init(void) struct attribute **attrs; int ret, i; - attrs = kmalloc_array(ARRAY_SIZE(paiext_ctrnames) + 1, sizeof(*attrs), - GFP_KERNEL); + attrs = kmalloc_array(paiext_cnt + 2, sizeof(*attrs), GFP_KERNEL); if (!attrs) return -ENOMEM; - for (i = 0; i < ARRAY_SIZE(paiext_ctrnames); i++) { + for (i = 0; i <= paiext_cnt; i++) { ret = attr_event_init_one(attrs, i); if (ret) { - attr_event_free(attrs, i - 1); + attr_event_free(attrs, i); return ret; } } diff --git a/arch/s390/kernel/perf_regs.c b/arch/s390/kernel/perf_regs.c index 3d93656bd948..a6b058ee4a36 100644 --- a/arch/s390/kernel/perf_regs.c +++ b/arch/s390/kernel/perf_regs.c @@ -5,8 +5,7 @@ #include <linux/errno.h> #include <linux/bug.h> #include <asm/ptrace.h> -#include <asm/fpu/api.h> -#include <asm/fpu/types.h> +#include <asm/fpu.h> u64 perf_reg_value(struct pt_regs *regs, int idx) { @@ -20,10 +19,7 @@ u64 perf_reg_value(struct pt_regs *regs, int idx) return 0; idx -= PERF_REG_S390_FP0; - if (cpu_has_vx()) - fp = *(freg_t *)(current->thread.fpu.vxrs + idx); - else - fp = current->thread.fpu.fprs[idx]; + fp = *(freg_t *)(current->thread.ufpu.vxrs + idx); return fp.ui; } @@ -65,6 +61,6 @@ void perf_get_regs_user(struct perf_regs *regs_user, */ regs_user->regs = task_pt_regs(current); if (user_mode(regs_user->regs)) - save_fpu_regs(); + save_user_fpu_regs(); regs_user->abi = perf_reg_abi(current); } diff --git a/arch/s390/kernel/process.c b/arch/s390/kernel/process.c index 4e3b366589fb..dd456b475861 100644 --- a/arch/s390/kernel/process.c +++ b/arch/s390/kernel/process.c @@ -31,15 +31,19 @@ #include <linux/init_task.h> #include <linux/entry-common.h> #include <linux/io.h> +#include <asm/guarded_storage.h> +#include <asm/access-regs.h> +#include <asm/switch_to.h> #include <asm/cpu_mf.h> #include <asm/processor.h> +#include <asm/ptrace.h> #include <asm/vtimer.h> #include <asm/exec.h> +#include <asm/fpu.h> #include <asm/irq.h> #include <asm/nmi.h> #include <asm/smp.h> #include <asm/stacktrace.h> -#include <asm/switch_to.h> #include <asm/runtime_instr.h> #include <asm/unwind.h> #include "entry.h" @@ -84,13 +88,13 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) { /* * Save the floating-point or vector register state of the current - * task and set the CIF_FPU flag to lazy restore the FPU register + * task and set the TIF_FPU flag to lazy restore the FPU register * state when returning to user space. */ - save_fpu_regs(); + save_user_fpu_regs(); *dst = *src; - dst->thread.fpu.regs = dst->thread.fpu.fprs; + dst->thread.kfpu_flags = 0; /* * Don't transfer over the runtime instrumentation or the guarded @@ -186,8 +190,23 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) void execve_tail(void) { - current->thread.fpu.fpc = 0; - asm volatile("sfpc %0" : : "d" (0)); + current->thread.ufpu.fpc = 0; + fpu_sfpc(0); +} + +struct task_struct *__switch_to(struct task_struct *prev, struct task_struct *next) +{ + save_user_fpu_regs(); + save_kernel_fpu_regs(&prev->thread); + save_access_regs(&prev->thread.acrs[0]); + save_ri_cb(prev->thread.ri_cb); + save_gs_cb(prev->thread.gs_cb); + update_cr_regs(next); + restore_kernel_fpu_regs(&next->thread); + restore_access_regs(&next->thread.acrs[0]); + restore_ri_cb(next->thread.ri_cb, prev->thread.ri_cb); + restore_gs_cb(next->thread.gs_cb); + return __switch_to_asm(prev, next); } unsigned long __get_wchan(struct task_struct *p) diff --git a/arch/s390/kernel/ptrace.c b/arch/s390/kernel/ptrace.c index f1897a8bb221..1cfed8b710b8 100644 --- a/arch/s390/kernel/ptrace.c +++ b/arch/s390/kernel/ptrace.c @@ -24,13 +24,14 @@ #include <linux/seccomp.h> #include <linux/compat.h> #include <trace/syscall.h> +#include <asm/guarded_storage.h> +#include <asm/access-regs.h> #include <asm/page.h> #include <linux/uaccess.h> #include <asm/unistd.h> -#include <asm/switch_to.h> #include <asm/runtime_instr.h> #include <asm/facility.h> -#include <asm/fpu/api.h> +#include <asm/fpu.h> #include "entry.h" @@ -246,22 +247,15 @@ static unsigned long __peek_user(struct task_struct *child, addr_t addr) /* * floating point control reg. is in the thread structure */ - tmp = child->thread.fpu.fpc; + tmp = child->thread.ufpu.fpc; tmp <<= BITS_PER_LONG - 32; } else if (addr < offsetof(struct user, regs.fp_regs) + sizeof(s390_fp_regs)) { /* - * floating point regs. are either in child->thread.fpu - * or the child->thread.fpu.vxrs array + * floating point regs. are in the child->thread.ufpu.vxrs array */ offset = addr - offsetof(struct user, regs.fp_regs.fprs); - if (cpu_has_vx()) - tmp = *(addr_t *) - ((addr_t) child->thread.fpu.vxrs + 2*offset); - else - tmp = *(addr_t *) - ((addr_t) child->thread.fpu.fprs + offset); - + tmp = *(addr_t *)((addr_t)child->thread.ufpu.vxrs + 2 * offset); } else if (addr < offsetof(struct user, regs.per_info) + sizeof(per_struct)) { /* * Handle access to the per_info structure. @@ -395,21 +389,14 @@ static int __poke_user(struct task_struct *child, addr_t addr, addr_t data) */ if ((unsigned int)data != 0) return -EINVAL; - child->thread.fpu.fpc = data >> (BITS_PER_LONG - 32); + child->thread.ufpu.fpc = data >> (BITS_PER_LONG - 32); } else if (addr < offsetof(struct user, regs.fp_regs) + sizeof(s390_fp_regs)) { /* - * floating point regs. are either in child->thread.fpu - * or the child->thread.fpu.vxrs array + * floating point regs. are in the child->thread.ufpu.vxrs array */ offset = addr - offsetof(struct user, regs.fp_regs.fprs); - if (cpu_has_vx()) - *(addr_t *)((addr_t) - child->thread.fpu.vxrs + 2*offset) = data; - else - *(addr_t *)((addr_t) - child->thread.fpu.fprs + offset) = data; - + *(addr_t *)((addr_t)child->thread.ufpu.vxrs + 2 * offset) = data; } else if (addr < offsetof(struct user, regs.per_info) + sizeof(per_struct)) { /* * Handle access to the per_info structure. @@ -622,21 +609,14 @@ static u32 __peek_user_compat(struct task_struct *child, addr_t addr) /* * floating point control reg. is in the thread structure */ - tmp = child->thread.fpu.fpc; + tmp = child->thread.ufpu.fpc; } else if (addr < offsetof(struct compat_user, regs.fp_regs) + sizeof(s390_fp_regs)) { /* - * floating point regs. are either in child->thread.fpu - * or the child->thread.fpu.vxrs array + * floating point regs. are in the child->thread.ufpu.vxrs array */ offset = addr - offsetof(struct compat_user, regs.fp_regs.fprs); - if (cpu_has_vx()) - tmp = *(__u32 *) - ((addr_t) child->thread.fpu.vxrs + 2*offset); - else - tmp = *(__u32 *) - ((addr_t) child->thread.fpu.fprs + offset); - + tmp = *(__u32 *)((addr_t)child->thread.ufpu.vxrs + 2 * offset); } else if (addr < offsetof(struct compat_user, regs.per_info) + sizeof(struct compat_per_struct_kernel)) { /* * Handle access to the per_info structure. @@ -748,21 +728,14 @@ static int __poke_user_compat(struct task_struct *child, /* * floating point control reg. is in the thread structure */ - child->thread.fpu.fpc = data; + child->thread.ufpu.fpc = data; } else if (addr < offsetof(struct compat_user, regs.fp_regs) + sizeof(s390_fp_regs)) { /* - * floating point regs. are either in child->thread.fpu - * or the child->thread.fpu.vxrs array + * floating point regs. are in the child->thread.ufpu.vxrs array */ offset = addr - offsetof(struct compat_user, regs.fp_regs.fprs); - if (cpu_has_vx()) - *(__u32 *)((addr_t) - child->thread.fpu.vxrs + 2*offset) = tmp; - else - *(__u32 *)((addr_t) - child->thread.fpu.fprs + offset) = tmp; - + *(__u32 *)((addr_t)child->thread.ufpu.vxrs + 2 * offset) = tmp; } else if (addr < offsetof(struct compat_user, regs.per_info) + sizeof(struct compat_per_struct_kernel)) { /* * Handle access to the per_info structure. @@ -893,10 +866,10 @@ static int s390_fpregs_get(struct task_struct *target, _s390_fp_regs fp_regs; if (target == current) - save_fpu_regs(); + save_user_fpu_regs(); - fp_regs.fpc = target->thread.fpu.fpc; - fpregs_store(&fp_regs, &target->thread.fpu); + fp_regs.fpc = target->thread.ufpu.fpc; + fpregs_store(&fp_regs, &target->thread.ufpu); return membuf_write(&to, &fp_regs, sizeof(fp_regs)); } @@ -910,22 +883,17 @@ static int s390_fpregs_set(struct task_struct *target, freg_t fprs[__NUM_FPRS]; if (target == current) - save_fpu_regs(); - - if (cpu_has_vx()) - convert_vx_to_fp(fprs, target->thread.fpu.vxrs); - else - memcpy(&fprs, target->thread.fpu.fprs, sizeof(fprs)); - + save_user_fpu_regs(); + convert_vx_to_fp(fprs, target->thread.ufpu.vxrs); if (count > 0 && pos < offsetof(s390_fp_regs, fprs)) { - u32 ufpc[2] = { target->thread.fpu.fpc, 0 }; + u32 ufpc[2] = { target->thread.ufpu.fpc, 0 }; rc = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &ufpc, 0, offsetof(s390_fp_regs, fprs)); if (rc) return rc; if (ufpc[1] != 0) return -EINVAL; - target->thread.fpu.fpc = ufpc[0]; + target->thread.ufpu.fpc = ufpc[0]; } if (rc == 0 && count > 0) @@ -933,12 +901,7 @@ static int s390_fpregs_set(struct task_struct *target, fprs, offsetof(s390_fp_regs, fprs), -1); if (rc) return rc; - - if (cpu_has_vx()) - convert_fp_to_vx(target->thread.fpu.vxrs, fprs); - else - memcpy(target->thread.fpu.fprs, &fprs, sizeof(fprs)); - + convert_fp_to_vx(target->thread.ufpu.vxrs, fprs); return rc; } @@ -988,9 +951,9 @@ static int s390_vxrs_low_get(struct task_struct *target, if (!cpu_has_vx()) return -ENODEV; if (target == current) - save_fpu_regs(); + save_user_fpu_regs(); for (i = 0; i < __NUM_VXRS_LOW; i++) - vxrs[i] = target->thread.fpu.vxrs[i].low; + vxrs[i] = target->thread.ufpu.vxrs[i].low; return membuf_write(&to, vxrs, sizeof(vxrs)); } @@ -1005,15 +968,15 @@ static int s390_vxrs_low_set(struct task_struct *target, if (!cpu_has_vx()) return -ENODEV; if (target == current) - save_fpu_regs(); + save_user_fpu_regs(); for (i = 0; i < __NUM_VXRS_LOW; i++) - vxrs[i] = target->thread.fpu.vxrs[i].low; + vxrs[i] = target->thread.ufpu.vxrs[i].low; rc = user_regset_copyin(&pos, &count, &kbuf, &ubuf, vxrs, 0, -1); if (rc == 0) for (i = 0; i < __NUM_VXRS_LOW; i++) - target->thread.fpu.vxrs[i].low = vxrs[i]; + target->thread.ufpu.vxrs[i].low = vxrs[i]; return rc; } @@ -1025,8 +988,8 @@ static int s390_vxrs_high_get(struct task_struct *target, if (!cpu_has_vx()) return -ENODEV; if (target == current) - save_fpu_regs(); - return membuf_write(&to, target->thread.fpu.vxrs + __NUM_VXRS_LOW, + save_user_fpu_regs(); + return membuf_write(&to, target->thread.ufpu.vxrs + __NUM_VXRS_LOW, __NUM_VXRS_HIGH * sizeof(__vector128)); } @@ -1040,10 +1003,10 @@ static int s390_vxrs_high_set(struct task_struct *target, if (!cpu_has_vx()) return -ENODEV; if (target == current) - save_fpu_regs(); + save_user_fpu_regs(); rc = user_regset_copyin(&pos, &count, &kbuf, &ubuf, - target->thread.fpu.vxrs + __NUM_VXRS_LOW, 0, -1); + target->thread.ufpu.vxrs + __NUM_VXRS_LOW, 0, -1); return rc; } diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index d1f3b56e7afc..24ed33f044ec 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -504,12 +504,12 @@ static void __init setup_resources(void) int j; u64 i; - code_resource.start = (unsigned long) _text; - code_resource.end = (unsigned long) _etext - 1; - data_resource.start = (unsigned long) _etext; - data_resource.end = (unsigned long) _edata - 1; - bss_resource.start = (unsigned long) __bss_start; - bss_resource.end = (unsigned long) __bss_stop - 1; + code_resource.start = __pa_symbol(_text); + code_resource.end = __pa_symbol(_etext) - 1; + data_resource.start = __pa_symbol(_etext); + data_resource.end = __pa_symbol(_edata) - 1; + bss_resource.start = __pa_symbol(__bss_start); + bss_resource.end = __pa_symbol(__bss_stop) - 1; for_each_mem_range(i, &start, &end) { res = memblock_alloc(sizeof(*res), 8); diff --git a/arch/s390/kernel/signal.c b/arch/s390/kernel/signal.c index 43e9661cd715..6c2cb345402f 100644 --- a/arch/s390/kernel/signal.c +++ b/arch/s390/kernel/signal.c @@ -30,8 +30,8 @@ #include <linux/compat.h> #include <asm/ucontext.h> #include <linux/uaccess.h> +#include <asm/access-regs.h> #include <asm/lowcore.h> -#include <asm/switch_to.h> #include <asm/vdso.h> #include "entry.h" @@ -109,7 +109,7 @@ struct rt_sigframe static void store_sigregs(void) { save_access_regs(current->thread.acrs); - save_fpu_regs(); + save_user_fpu_regs(); } /* Load registers after signal return */ @@ -131,7 +131,7 @@ static int save_sigregs(struct pt_regs *regs, _sigregs __user *sregs) memcpy(&user_sregs.regs.gprs, ®s->gprs, sizeof(sregs->regs.gprs)); memcpy(&user_sregs.regs.acrs, current->thread.acrs, sizeof(user_sregs.regs.acrs)); - fpregs_store(&user_sregs.fpregs, ¤t->thread.fpu); + fpregs_store(&user_sregs.fpregs, ¤t->thread.ufpu); if (__copy_to_user(sregs, &user_sregs, sizeof(_sigregs))) return -EFAULT; return 0; @@ -165,7 +165,7 @@ static int restore_sigregs(struct pt_regs *regs, _sigregs __user *sregs) memcpy(¤t->thread.acrs, &user_sregs.regs.acrs, sizeof(current->thread.acrs)); - fpregs_load(&user_sregs.fpregs, ¤t->thread.fpu); + fpregs_load(&user_sregs.fpregs, ¤t->thread.ufpu); clear_pt_regs_flag(regs, PIF_SYSCALL); /* No longer in a system call */ return 0; @@ -181,11 +181,11 @@ static int save_sigregs_ext(struct pt_regs *regs, /* Save vector registers to signal stack */ if (cpu_has_vx()) { for (i = 0; i < __NUM_VXRS_LOW; i++) - vxrs[i] = current->thread.fpu.vxrs[i].low; + vxrs[i] = current->thread.ufpu.vxrs[i].low; if (__copy_to_user(&sregs_ext->vxrs_low, vxrs, sizeof(sregs_ext->vxrs_low)) || __copy_to_user(&sregs_ext->vxrs_high, - current->thread.fpu.vxrs + __NUM_VXRS_LOW, + current->thread.ufpu.vxrs + __NUM_VXRS_LOW, sizeof(sregs_ext->vxrs_high))) return -EFAULT; } @@ -202,12 +202,12 @@ static int restore_sigregs_ext(struct pt_regs *regs, if (cpu_has_vx()) { if (__copy_from_user(vxrs, &sregs_ext->vxrs_low, sizeof(sregs_ext->vxrs_low)) || - __copy_from_user(current->thread.fpu.vxrs + __NUM_VXRS_LOW, + __copy_from_user(current->thread.ufpu.vxrs + __NUM_VXRS_LOW, &sregs_ext->vxrs_high, sizeof(sregs_ext->vxrs_high))) return -EFAULT; for (i = 0; i < __NUM_VXRS_LOW; i++) - current->thread.fpu.vxrs[i].low = vxrs[i]; + current->thread.ufpu.vxrs[i].low = vxrs[i]; } return 0; } @@ -222,7 +222,7 @@ SYSCALL_DEFINE0(sigreturn) if (__copy_from_user(&set.sig, &frame->sc.oldmask, _SIGMASK_COPY_SIZE)) goto badframe; set_current_blocked(&set); - save_fpu_regs(); + save_user_fpu_regs(); if (restore_sigregs(regs, &frame->sregs)) goto badframe; if (restore_sigregs_ext(regs, &frame->sregs_ext)) @@ -246,7 +246,7 @@ SYSCALL_DEFINE0(rt_sigreturn) set_current_blocked(&set); if (restore_altstack(&frame->uc.uc_stack)) goto badframe; - save_fpu_regs(); + save_user_fpu_regs(); if (restore_sigregs(regs, &frame->uc.uc_mcontext)) goto badframe; if (restore_sigregs_ext(regs, &frame->uc.uc_mcontext_ext)) diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index c39d9f0d4b1c..0324649aae0a 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -36,12 +36,13 @@ #include <linux/sched/task_stack.h> #include <linux/crash_dump.h> #include <linux/kprobes.h> +#include <asm/access-regs.h> #include <asm/asm-offsets.h> #include <asm/ctlreg.h> #include <asm/pfault.h> #include <asm/diag.h> -#include <asm/switch_to.h> #include <asm/facility.h> +#include <asm/fpu.h> #include <asm/ipl.h> #include <asm/setup.h> #include <asm/irq.h> diff --git a/arch/s390/kernel/sysinfo.c b/arch/s390/kernel/sysinfo.c index f6f8f498c9be..1b1be3110cfc 100644 --- a/arch/s390/kernel/sysinfo.c +++ b/arch/s390/kernel/sysinfo.c @@ -20,7 +20,7 @@ #include <asm/sysinfo.h> #include <asm/cpcmd.h> #include <asm/topology.h> -#include <asm/fpu/api.h> +#include <asm/fpu.h> int topology_max_mnest; @@ -426,9 +426,9 @@ subsys_initcall(create_proc_service_level); */ void s390_adjust_jiffies(void) { + DECLARE_KERNEL_FPU_ONSTACK16(fpu); struct sysinfo_1_2_2 *info; unsigned long capability; - struct kernel_fpu fpu; info = (void *) get_zeroed_page(GFP_KERNEL); if (!info) @@ -447,21 +447,14 @@ void s390_adjust_jiffies(void) * point division .. */ kernel_fpu_begin(&fpu, KERNEL_FPR); - asm volatile( - " sfpc %3\n" - " l %0,%1\n" - " tmlh %0,0xff80\n" - " jnz 0f\n" - " cefbr %%f2,%0\n" - " j 1f\n" - "0: le %%f2,%1\n" - "1: cefbr %%f0,%2\n" - " debr %%f0,%%f2\n" - " cgebr %0,5,%%f0\n" - : "=&d" (capability) - : "Q" (info->capability), "d" (10000000), "d" (0) - : "cc" - ); + fpu_sfpc(0); + if (info->capability & 0xff800000) + fpu_ldgr(2, info->capability); + else + fpu_cefbr(2, info->capability); + fpu_cefbr(0, 10000000); + fpu_debr(0, 2); + capability = fpu_cgebr(0, 5); kernel_fpu_end(&fpu, KERNEL_FPR); } else /* diff --git a/arch/s390/kernel/text_amode31.S b/arch/s390/kernel/text_amode31.S index 14c6d25c035f..c0a70efa2426 100644 --- a/arch/s390/kernel/text_amode31.S +++ b/arch/s390/kernel/text_amode31.S @@ -90,7 +90,7 @@ SYM_FUNC_START(_diag26c_amode31) SYM_FUNC_END(_diag26c_amode31) /* - * void _diag0c_amode31(struct hypfs_diag0c_entry *entry) + * void _diag0c_amode31(unsigned long rx) */ SYM_FUNC_START(_diag0c_amode31) sam31 diff --git a/arch/s390/kernel/time.c b/arch/s390/kernel/time.c index 14abad953c02..fb9f31f36628 100644 --- a/arch/s390/kernel/time.c +++ b/arch/s390/kernel/time.c @@ -251,8 +251,8 @@ static struct clocksource clocksource_tod = { .rating = 400, .read = read_tod_clock, .mask = CLOCKSOURCE_MASK(64), - .mult = 1000, - .shift = 12, + .mult = 4096000, + .shift = 24, .flags = CLOCK_SOURCE_IS_CONTINUOUS, .vdso_clock_mode = VDSO_CLOCKMODE_TOD, }; @@ -716,7 +716,7 @@ out_unlock: /* * STP subsys sysfs interface functions */ -static struct bus_type stp_subsys = { +static const struct bus_type stp_subsys = { .name = "stp", .dev_name = "stp", }; diff --git a/arch/s390/kernel/traps.c b/arch/s390/kernel/traps.c index 46dac4540ca8..52578b5cecbd 100644 --- a/arch/s390/kernel/traps.c +++ b/arch/s390/kernel/traps.c @@ -28,8 +28,8 @@ #include <linux/cpu.h> #include <linux/entry-common.h> #include <asm/asm-extable.h> -#include <asm/fpu/api.h> #include <asm/vtime.h> +#include <asm/fpu.h> #include "entry.h" static inline void __user *get_trap_ip(struct pt_regs *regs) @@ -201,8 +201,8 @@ static void vector_exception(struct pt_regs *regs) } /* get vector interrupt code from fpc */ - save_fpu_regs(); - vic = (current->thread.fpu.fpc & 0xf00) >> 8; + save_user_fpu_regs(); + vic = (current->thread.ufpu.fpc & 0xf00) >> 8; switch (vic) { case 1: /* invalid vector operation */ si_code = FPE_FLTINV; @@ -227,9 +227,9 @@ static void vector_exception(struct pt_regs *regs) static void data_exception(struct pt_regs *regs) { - save_fpu_regs(); - if (current->thread.fpu.fpc & FPC_DXC_MASK) - do_fp_trap(regs, current->thread.fpu.fpc); + save_user_fpu_regs(); + if (current->thread.ufpu.fpc & FPC_DXC_MASK) + do_fp_trap(regs, current->thread.ufpu.fpc); else do_trap(regs, SIGILL, ILL_ILLOPN, "data exception"); } diff --git a/arch/s390/kernel/uprobes.c b/arch/s390/kernel/uprobes.c index b88345ef8bd9..5b0633ea8d93 100644 --- a/arch/s390/kernel/uprobes.c +++ b/arch/s390/kernel/uprobes.c @@ -12,7 +12,6 @@ #include <linux/kdebug.h> #include <linux/sched/task_stack.h> -#include <asm/switch_to.h> #include <asm/facility.h> #include <asm/kprobes.h> #include <asm/dis.h> diff --git a/arch/s390/kernel/vdso32/Makefile b/arch/s390/kernel/vdso32/Makefile index caec7db6f966..b12a274cbb47 100644 --- a/arch/s390/kernel/vdso32/Makefile +++ b/arch/s390/kernel/vdso32/Makefile @@ -22,7 +22,7 @@ KBUILD_CFLAGS_32 := $(filter-out -m64,$(KBUILD_CFLAGS)) KBUILD_CFLAGS_32 := $(filter-out -mno-pic-data-is-text-relative,$(KBUILD_CFLAGS_32)) KBUILD_CFLAGS_32 += -m31 -fPIC -shared -fno-common -fno-builtin -LDFLAGS_vdso32.so.dbg += -fPIC -shared -soname=linux-vdso32.so.1 \ +LDFLAGS_vdso32.so.dbg += -shared -soname=linux-vdso32.so.1 \ --hash-style=both --build-id=sha1 -melf_s390 -T $(targets:%=$(obj)/%.dbg): KBUILD_CFLAGS = $(KBUILD_CFLAGS_32) diff --git a/arch/s390/kernel/vdso32/vdso32.lds.S b/arch/s390/kernel/vdso32/vdso32.lds.S index edf5ff1debe1..65b9513a5a0e 100644 --- a/arch/s390/kernel/vdso32/vdso32.lds.S +++ b/arch/s390/kernel/vdso32/vdso32.lds.S @@ -9,7 +9,6 @@ OUTPUT_FORMAT("elf32-s390", "elf32-s390", "elf32-s390") OUTPUT_ARCH(s390:31-bit) -ENTRY(_start) SECTIONS { diff --git a/arch/s390/kernel/vdso64/Makefile b/arch/s390/kernel/vdso64/Makefile index e3c9085f8fa7..ef9832726097 100644 --- a/arch/s390/kernel/vdso64/Makefile +++ b/arch/s390/kernel/vdso64/Makefile @@ -25,8 +25,9 @@ KBUILD_AFLAGS_64 += -m64 KBUILD_CFLAGS_64 := $(filter-out -m64,$(KBUILD_CFLAGS)) KBUILD_CFLAGS_64 := $(filter-out -mno-pic-data-is-text-relative,$(KBUILD_CFLAGS_64)) +KBUILD_CFLAGS_64 := $(filter-out -munaligned-symbols,$(KBUILD_CFLAGS_64)) KBUILD_CFLAGS_64 += -m64 -fPIC -fno-common -fno-builtin -ldflags-y := -fPIC -shared -soname=linux-vdso64.so.1 \ +ldflags-y := -shared -soname=linux-vdso64.so.1 \ --hash-style=both --build-id=sha1 -T $(targets:%=$(obj)/%.dbg): KBUILD_CFLAGS = $(KBUILD_CFLAGS_64) diff --git a/arch/s390/kernel/vdso64/vdso64.lds.S b/arch/s390/kernel/vdso64/vdso64.lds.S index 4461ea151e49..37e2a505e81d 100644 --- a/arch/s390/kernel/vdso64/vdso64.lds.S +++ b/arch/s390/kernel/vdso64/vdso64.lds.S @@ -9,7 +9,6 @@ OUTPUT_FORMAT("elf64-s390", "elf64-s390", "elf64-s390") OUTPUT_ARCH(s390:64-bit) -ENTRY(_start) SECTIONS { diff --git a/arch/s390/kernel/vmlinux.lds.S b/arch/s390/kernel/vmlinux.lds.S index e32ef446f451..48de296e8905 100644 --- a/arch/s390/kernel/vmlinux.lds.S +++ b/arch/s390/kernel/vmlinux.lds.S @@ -59,6 +59,14 @@ SECTIONS } :text = 0x0700 RO_DATA(PAGE_SIZE) + .data.rel.ro : { + *(.data.rel.ro .data.rel.ro.*) + } + .got : { + __got_start = .; + *(.got) + __got_end = .; + } . = ALIGN(PAGE_SIZE); _sdata = .; /* Start of data section */ @@ -73,6 +81,9 @@ SECTIONS __end_ro_after_init = .; RW_DATA(0x100, PAGE_SIZE, THREAD_SIZE) + .data.rel : { + *(.data.rel*) + } BOOT_DATA_PRESERVED . = ALIGN(8); @@ -181,6 +192,7 @@ SECTIONS PERCPU_SECTION(0x100) +#ifdef CONFIG_PIE_BUILD .dynsym ALIGN(8) : { __dynsym_start = .; *(.dynsym) @@ -191,6 +203,19 @@ SECTIONS *(.rela*) __rela_dyn_end = .; } + .dynamic ALIGN(8) : { + *(.dynamic) + } + .dynstr ALIGN(8) : { + *(.dynstr) + } +#endif + .hash ALIGN(8) : { + *(.hash) + } + .gnu.hash ALIGN(8) : { + *(.gnu.hash) + } . = ALIGN(PAGE_SIZE); __init_end = .; /* freed after init ends here */ @@ -214,9 +239,14 @@ SECTIONS QUAD(__boot_data_preserved_start) /* bootdata_preserved_off */ QUAD(__boot_data_preserved_end - __boot_data_preserved_start) /* bootdata_preserved_size */ +#ifdef CONFIG_PIE_BUILD QUAD(__dynsym_start) /* dynsym_start */ QUAD(__rela_dyn_start) /* rela_dyn_start */ QUAD(__rela_dyn_end) /* rela_dyn_end */ +#else + QUAD(__got_start) /* got_start */ + QUAD(__got_end) /* got_end */ +#endif QUAD(_eamode31 - _samode31) /* amode31_size */ QUAD(init_mm) QUAD(swapper_pg_dir) @@ -235,6 +265,30 @@ SECTIONS DWARF_DEBUG ELF_DETAILS + /* + * Make sure that the .got.plt is either completely empty or it + * contains only the three reserved double words. + */ + .got.plt : { + *(.got.plt) + } + ASSERT(SIZEOF(.got.plt) == 0 || SIZEOF(.got.plt) == 0x18, "Unexpected GOT/PLT entries detected!") + + /* + * Sections that should stay zero sized, which is safer to + * explicitly check instead of blindly discarding. + */ + .plt : { + *(.plt) *(.plt.*) *(.iplt) *(.igot .igot.plt) + } + ASSERT(SIZEOF(.plt) == 0, "Unexpected run-time procedure linkages detected!") +#ifndef CONFIG_PIE_BUILD + .rela.dyn : { + *(.rela.*) *(.rela_*) + } + ASSERT(SIZEOF(.rela.dyn) == 0, "Unexpected run-time relocations (.rela) detected!") +#endif + /* Sections to be discarded */ DISCARDS /DISCARD/ : { diff --git a/arch/s390/kvm/gaccess.c b/arch/s390/kvm/gaccess.c index 5bfcc50c1a68..ee863566910b 100644 --- a/arch/s390/kvm/gaccess.c +++ b/arch/s390/kvm/gaccess.c @@ -11,11 +11,11 @@ #include <linux/err.h> #include <linux/pgtable.h> #include <linux/bitfield.h> +#include <asm/access-regs.h> #include <asm/fault.h> #include <asm/gmap.h> #include "kvm-s390.h" #include "gaccess.h" -#include <asm/switch_to.h> union asce { unsigned long val; @@ -391,7 +391,8 @@ static int ar_translation(struct kvm_vcpu *vcpu, union asce *asce, u8 ar, if (ar >= NUM_ACRS) return -EINVAL; - save_access_regs(vcpu->run->s.regs.acrs); + if (vcpu->arch.acrs_loaded) + save_access_regs(vcpu->run->s.regs.acrs); alet.val = vcpu->run->s.regs.acrs[ar]; if (ar == 0 || alet.val == 0) { diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/interrupt.c index fc4007cc067a..dc721d50a942 100644 --- a/arch/s390/kvm/interrupt.c +++ b/arch/s390/kvm/interrupt.c @@ -19,13 +19,13 @@ #include <linux/slab.h> #include <linux/bitmap.h> #include <linux/vmalloc.h> +#include <asm/access-regs.h> #include <asm/asm-offsets.h> #include <asm/dis.h> #include <linux/uaccess.h> #include <asm/sclp.h> #include <asm/isc.h> #include <asm/gmap.h> -#include <asm/switch_to.h> #include <asm/nmi.h> #include <asm/airq.h> #include <asm/tpi.h> @@ -584,7 +584,7 @@ static int __write_machine_check(struct kvm_vcpu *vcpu, mci.val = mchk->mcic; /* take care of lazy register loading */ - save_fpu_regs(); + kvm_s390_fpu_store(vcpu->run); save_access_regs(vcpu->run->s.regs.acrs); if (MACHINE_HAS_GS && vcpu->arch.gs_enabled) save_gs_cb(current->thread.gs_cb); @@ -648,7 +648,7 @@ static int __write_machine_check(struct kvm_vcpu *vcpu, } rc |= write_guest_lc(vcpu, __LC_GPREGS_SAVE_AREA, vcpu->run->s.regs.gprs, 128); - rc |= put_guest_lc(vcpu, current->thread.fpu.fpc, + rc |= put_guest_lc(vcpu, vcpu->run->s.regs.fpc, (u32 __user *) __LC_FP_CREG_SAVE_AREA); rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->todpr, (u32 __user *) __LC_TOD_PROGREG_SAVE_AREA); diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index ea63ac769889..b11bb8e780a1 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -33,19 +33,19 @@ #include <linux/pgtable.h> #include <linux/mmu_notifier.h> +#include <asm/access-regs.h> #include <asm/asm-offsets.h> #include <asm/lowcore.h> #include <asm/stp.h> #include <asm/gmap.h> #include <asm/nmi.h> -#include <asm/switch_to.h> #include <asm/isc.h> #include <asm/sclp.h> #include <asm/cpacf.h> #include <asm/timex.h> +#include <asm/fpu.h> #include <asm/ap.h> #include <asm/uv.h> -#include <asm/fpu/api.h> #include "kvm-s390.h" #include "gaccess.h" #include "pci.h" @@ -3951,6 +3951,7 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) KVM_SYNC_ARCH0 | KVM_SYNC_PFAULT | KVM_SYNC_DIAG318; + vcpu->arch.acrs_loaded = false; kvm_s390_set_prefix(vcpu, 0); if (test_kvm_facility(vcpu->kvm, 64)) vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB; @@ -4829,8 +4830,6 @@ static int __vcpu_run(struct kvm_vcpu *vcpu) vcpu->run->s.regs.gprs, sizeof(sie_page->pv_grregs)); } - if (test_cpu_flag(CIF_FPU)) - load_fpu_regs(); exit_reason = sie64a(vcpu->arch.sie_block, vcpu->run->s.regs.gprs); if (kvm_s390_pv_cpu_is_protected(vcpu)) { @@ -4951,16 +4950,8 @@ static void sync_regs(struct kvm_vcpu *vcpu) } save_access_regs(vcpu->arch.host_acrs); restore_access_regs(vcpu->run->s.regs.acrs); - /* save host (userspace) fprs/vrs */ - save_fpu_regs(); - vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc; - vcpu->arch.host_fpregs.regs = current->thread.fpu.regs; - if (cpu_has_vx()) - current->thread.fpu.regs = vcpu->run->s.regs.vrs; - else - current->thread.fpu.regs = vcpu->run->s.regs.fprs; - current->thread.fpu.fpc = vcpu->run->s.regs.fpc; - + vcpu->arch.acrs_loaded = true; + kvm_s390_fpu_load(vcpu->run); /* Sync fmt2 only data */ if (likely(!kvm_s390_pv_cpu_is_protected(vcpu))) { sync_regs_fmt2(vcpu); @@ -5021,12 +5012,8 @@ static void store_regs(struct kvm_vcpu *vcpu) kvm_run->s.regs.pfc = vcpu->arch.pfault_compare; save_access_regs(vcpu->run->s.regs.acrs); restore_access_regs(vcpu->arch.host_acrs); - /* Save guest register state */ - save_fpu_regs(); - vcpu->run->s.regs.fpc = current->thread.fpu.fpc; - /* Restore will be done lazily at return */ - current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc; - current->thread.fpu.regs = vcpu->arch.host_fpregs.regs; + vcpu->arch.acrs_loaded = false; + kvm_s390_fpu_store(vcpu->run); if (likely(!kvm_s390_pv_cpu_is_protected(vcpu))) store_regs_fmt2(vcpu); } @@ -5034,6 +5021,7 @@ static void store_regs(struct kvm_vcpu *vcpu) int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) { struct kvm_run *kvm_run = vcpu->run; + DECLARE_KERNEL_FPU_ONSTACK32(fpu); int rc; /* @@ -5075,6 +5063,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) goto out; } + kernel_fpu_begin(&fpu, KERNEL_FPC | KERNEL_VXR); sync_regs(vcpu); enable_cpu_timer_accounting(vcpu); @@ -5098,6 +5087,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) disable_cpu_timer_accounting(vcpu); store_regs(vcpu); + kernel_fpu_end(&fpu, KERNEL_FPC | KERNEL_VXR); kvm_sigset_deactivate(vcpu); @@ -5172,8 +5162,7 @@ int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr) * switch in the run ioctl. Let's update our copies before we save * it into the save area */ - save_fpu_regs(); - vcpu->run->s.regs.fpc = current->thread.fpu.fpc; + kvm_s390_fpu_store(vcpu->run); save_access_regs(vcpu->run->s.regs.acrs); return kvm_s390_store_status_unloaded(vcpu, addr); diff --git a/arch/s390/kvm/kvm-s390.h b/arch/s390/kvm/kvm-s390.h index a7ea80cfa445..111eb5c74784 100644 --- a/arch/s390/kvm/kvm-s390.h +++ b/arch/s390/kvm/kvm-s390.h @@ -20,6 +20,24 @@ #include <asm/processor.h> #include <asm/sclp.h> +static inline void kvm_s390_fpu_store(struct kvm_run *run) +{ + fpu_stfpc(&run->s.regs.fpc); + if (cpu_has_vx()) + save_vx_regs((__vector128 *)&run->s.regs.vrs); + else + save_fp_regs((freg_t *)&run->s.regs.fprs); +} + +static inline void kvm_s390_fpu_load(struct kvm_run *run) +{ + fpu_lfpc_safe(&run->s.regs.fpc); + if (cpu_has_vx()) + load_vx_regs((__vector128 *)&run->s.regs.vrs); + else + load_fp_regs((freg_t *)&run->s.regs.fprs); +} + /* Transactional Memory Execution related macros */ #define IS_TE_ENABLED(vcpu) ((vcpu->arch.sie_block->ecb & ECB_TE)) #define TDB_FORMAT1 1 diff --git a/arch/s390/kvm/vsie.c b/arch/s390/kvm/vsie.c index 3af3bd20ac7b..b2c9f010f0fe 100644 --- a/arch/s390/kvm/vsie.c +++ b/arch/s390/kvm/vsie.c @@ -18,7 +18,6 @@ #include <asm/sclp.h> #include <asm/nmi.h> #include <asm/dis.h> -#include <asm/fpu/api.h> #include <asm/facility.h> #include "kvm-s390.h" #include "gaccess.h" @@ -1149,8 +1148,6 @@ static int do_vsie_run(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page) */ vcpu->arch.sie_block->prog0c |= PROG_IN_SIE; barrier(); - if (test_cpu_flag(CIF_FPU)) - load_fpu_regs(); if (!kvm_s390_vcpu_sie_inhibited(vcpu)) rc = sie64a(scb_s, vcpu->run->s.regs.gprs); barrier(); diff --git a/arch/s390/lib/Makefile b/arch/s390/lib/Makefile index 7c50eca85ca4..90eac15ea62a 100644 --- a/arch/s390/lib/Makefile +++ b/arch/s390/lib/Makefile @@ -4,6 +4,7 @@ # lib-y += delay.o string.o uaccess.o find.o spinlock.o tishift.o +lib-y += csum-partial.o obj-y += mem.o xor.o lib-$(CONFIG_KPROBES) += probes.o lib-$(CONFIG_UPROBES) += probes.o diff --git a/arch/s390/lib/csum-partial.c b/arch/s390/lib/csum-partial.c new file mode 100644 index 000000000000..458abd9bac70 --- /dev/null +++ b/arch/s390/lib/csum-partial.c @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/export.h> +#include <asm/checksum.h> +#include <asm/fpu.h> + +/* + * Computes the checksum of a memory block at src, length len, + * and adds in "sum" (32-bit). If copy is true copies to dst. + * + * Returns a 32-bit number suitable for feeding into itself + * or csum_tcpudp_magic. + * + * This function must be called with even lengths, except + * for the last fragment, which may be odd. + * + * It's best to have src and dst aligned on a 64-bit boundary. + */ +static __always_inline __wsum csum_copy(void *dst, const void *src, int len, __wsum sum, bool copy) +{ + DECLARE_KERNEL_FPU_ONSTACK8(vxstate); + + if (!cpu_has_vx()) { + if (copy) + memcpy(dst, src, len); + return cksm(dst, len, sum); + } + kernel_fpu_begin(&vxstate, KERNEL_VXR_V16V23); + fpu_vlvgf(16, (__force u32)sum, 1); + fpu_vzero(17); + fpu_vzero(18); + fpu_vzero(19); + while (len >= 64) { + fpu_vlm(20, 23, src); + if (copy) { + fpu_vstm(20, 23, dst); + dst += 64; + } + fpu_vcksm(16, 20, 16); + fpu_vcksm(17, 21, 17); + fpu_vcksm(18, 22, 18); + fpu_vcksm(19, 23, 19); + src += 64; + len -= 64; + } + while (len >= 32) { + fpu_vlm(20, 21, src); + if (copy) { + fpu_vstm(20, 21, dst); + dst += 32; + } + fpu_vcksm(16, 20, 16); + fpu_vcksm(17, 21, 17); + src += 32; + len -= 32; + } + while (len >= 16) { + fpu_vl(20, src); + if (copy) { + fpu_vst(20, dst); + dst += 16; + } + fpu_vcksm(16, 20, 16); + src += 16; + len -= 16; + } + if (len) { + fpu_vll(20, len - 1, src); + if (copy) + fpu_vstl(20, len - 1, dst); + fpu_vcksm(16, 20, 16); + } + fpu_vcksm(18, 19, 18); + fpu_vcksm(16, 17, 16); + fpu_vcksm(16, 18, 16); + sum = (__force __wsum)fpu_vlgvf(16, 1); + kernel_fpu_end(&vxstate, KERNEL_VXR_V16V23); + return sum; +} + +__wsum csum_partial(const void *buff, int len, __wsum sum) +{ + return csum_copy(NULL, buff, len, sum, false); +} +EXPORT_SYMBOL(csum_partial); + +__wsum csum_partial_copy_nocheck(const void *src, void *dst, int len) +{ + return csum_copy(dst, src, len, 0, true); +} +EXPORT_SYMBOL(csum_partial_copy_nocheck); diff --git a/arch/s390/mm/extmem.c b/arch/s390/mm/extmem.c index e41869f5cc95..282fefe107a2 100644 --- a/arch/s390/mm/extmem.c +++ b/arch/s390/mm/extmem.c @@ -136,7 +136,7 @@ dcss_diag(int *func, void *parameter, unsigned long rx, ry; int rc; - rx = (unsigned long) parameter; + rx = virt_to_phys(parameter); ry = (unsigned long) *func; diag_stat_inc(DIAG_STAT_X064); @@ -178,7 +178,7 @@ query_segment_type (struct dcss_segment *seg) /* initialize diag input parameters */ qin->qopcode = DCSS_FINDSEGA; - qin->qoutptr = (unsigned long) qout; + qin->qoutptr = virt_to_phys(qout); qin->qoutlen = sizeof(struct qout64); memcpy (qin->qname, seg->dcss_name, 8); diff --git a/arch/s390/mm/mmap.c b/arch/s390/mm/mmap.c index fc9a7dc26c5e..b14fc0887654 100644 --- a/arch/s390/mm/mmap.c +++ b/arch/s390/mm/mmap.c @@ -71,6 +71,15 @@ static inline unsigned long mmap_base(unsigned long rnd, return PAGE_ALIGN(STACK_TOP - gap - rnd); } +static int get_align_mask(struct file *filp, unsigned long flags) +{ + if (!(current->flags & PF_RANDOMIZE)) + return 0; + if (filp || (flags & MAP_SHARED)) + return MMAP_ALIGN_MASK << PAGE_SHIFT; + return 0; +} + unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags) @@ -97,10 +106,7 @@ unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, info.length = len; info.low_limit = mm->mmap_base; info.high_limit = TASK_SIZE; - if (filp || (flags & MAP_SHARED)) - info.align_mask = MMAP_ALIGN_MASK << PAGE_SHIFT; - else - info.align_mask = 0; + info.align_mask = get_align_mask(filp, flags); info.align_offset = pgoff << PAGE_SHIFT; addr = vm_unmapped_area(&info); if (offset_in_page(addr)) @@ -138,10 +144,7 @@ unsigned long arch_get_unmapped_area_topdown(struct file *filp, unsigned long ad info.length = len; info.low_limit = PAGE_SIZE; info.high_limit = mm->mmap_base; - if (filp || (flags & MAP_SHARED)) - info.align_mask = MMAP_ALIGN_MASK << PAGE_SHIFT; - else - info.align_mask = 0; + info.align_mask = get_align_mask(filp, flags); info.align_offset = pgoff << PAGE_SHIFT; addr = vm_unmapped_area(&info); diff --git a/arch/s390/pci/pci.c b/arch/s390/pci/pci.c index 52a44e353796..26afde0d1ed3 100644 --- a/arch/s390/pci/pci.c +++ b/arch/s390/pci/pci.c @@ -28,6 +28,7 @@ #include <linux/jump_label.h> #include <linux/pci.h> #include <linux/printk.h> +#include <linux/lockdep.h> #include <asm/isc.h> #include <asm/airq.h> @@ -730,12 +731,12 @@ EXPORT_SYMBOL_GPL(zpci_disable_device); * equivalent to its state during boot when first probing a driver. * Consequently after reset the PCI function requires re-initialization via the * common PCI code including re-enabling IRQs via pci_alloc_irq_vectors() - * and enabling the function via e.g.pci_enablde_device_flags().The caller + * and enabling the function via e.g. pci_enable_device_flags(). The caller * must guard against concurrent reset attempts. * * In most cases this function should not be called directly but through * pci_reset_function() or pci_reset_bus() which handle the save/restore and - * locking. + * locking - asserted by lockdep. * * Return: 0 on success and an error value otherwise */ @@ -744,6 +745,7 @@ int zpci_hot_reset_device(struct zpci_dev *zdev) u8 status; int rc; + lockdep_assert_held(&zdev->state_lock); zpci_dbg(3, "rst fid:%x, fh:%x\n", zdev->fid, zdev->fh); if (zdev_enabled(zdev)) { /* Disables device access, DMAs and IRQs (reset state) */ @@ -806,7 +808,8 @@ struct zpci_dev *zpci_create_device(u32 fid, u32 fh, enum zpci_state state) zdev->state = state; kref_init(&zdev->kref); - mutex_init(&zdev->lock); + mutex_init(&zdev->state_lock); + mutex_init(&zdev->fmb_lock); mutex_init(&zdev->kzdev_lock); rc = zpci_init_iommu(zdev); @@ -870,6 +873,10 @@ int zpci_deconfigure_device(struct zpci_dev *zdev) { int rc; + lockdep_assert_held(&zdev->state_lock); + if (zdev->state != ZPCI_FN_STATE_CONFIGURED) + return 0; + if (zdev->zbus->bus) zpci_bus_remove_device(zdev, false); @@ -889,7 +896,7 @@ int zpci_deconfigure_device(struct zpci_dev *zdev) } /** - * zpci_device_reserved() - Mark device as resverved + * zpci_device_reserved() - Mark device as reserved * @zdev: the zpci_dev that was reserved * * Handle the case that a given zPCI function was reserved by another system. @@ -899,8 +906,6 @@ int zpci_deconfigure_device(struct zpci_dev *zdev) */ void zpci_device_reserved(struct zpci_dev *zdev) { - if (zdev->has_hp_slot) - zpci_exit_slot(zdev); /* * Remove device from zpci_list as it is going away. This also * makes sure we ignore subsequent zPCI events for this device. @@ -918,6 +923,9 @@ void zpci_release_device(struct kref *kref) struct zpci_dev *zdev = container_of(kref, struct zpci_dev, kref); int ret; + if (zdev->has_hp_slot) + zpci_exit_slot(zdev); + if (zdev->zbus->bus) zpci_bus_remove_device(zdev, false); diff --git a/arch/s390/pci/pci_debug.c b/arch/s390/pci/pci_debug.c index 6dde2263c79d..2cb5043a997d 100644 --- a/arch/s390/pci/pci_debug.c +++ b/arch/s390/pci/pci_debug.c @@ -91,9 +91,9 @@ static int pci_perf_show(struct seq_file *m, void *v) if (!zdev) return 0; - mutex_lock(&zdev->lock); + mutex_lock(&zdev->fmb_lock); if (!zdev->fmb) { - mutex_unlock(&zdev->lock); + mutex_unlock(&zdev->fmb_lock); seq_puts(m, "FMB statistics disabled\n"); return 0; } @@ -130,7 +130,7 @@ static int pci_perf_show(struct seq_file *m, void *v) } pci_sw_counter_show(m); - mutex_unlock(&zdev->lock); + mutex_unlock(&zdev->fmb_lock); return 0; } @@ -148,7 +148,7 @@ static ssize_t pci_perf_seq_write(struct file *file, const char __user *ubuf, if (rc) return rc; - mutex_lock(&zdev->lock); + mutex_lock(&zdev->fmb_lock); switch (val) { case 0: rc = zpci_fmb_disable_device(zdev); @@ -157,7 +157,7 @@ static ssize_t pci_perf_seq_write(struct file *file, const char __user *ubuf, rc = zpci_fmb_enable_device(zdev); break; } - mutex_unlock(&zdev->lock); + mutex_unlock(&zdev->fmb_lock); return rc ? rc : count; } diff --git a/arch/s390/pci/pci_event.c b/arch/s390/pci/pci_event.c index 4d9773ef9e0a..dbe95ec5917e 100644 --- a/arch/s390/pci/pci_event.c +++ b/arch/s390/pci/pci_event.c @@ -267,6 +267,7 @@ static void __zpci_event_error(struct zpci_ccdf_err *ccdf) zpci_err_hex(ccdf, sizeof(*ccdf)); if (zdev) { + mutex_lock(&zdev->state_lock); zpci_update_fh(zdev, ccdf->fh); if (zdev->zbus->bus) pdev = pci_get_slot(zdev->zbus->bus, zdev->devfn); @@ -294,6 +295,8 @@ static void __zpci_event_error(struct zpci_ccdf_err *ccdf) } pci_dev_put(pdev); no_pdev: + if (zdev) + mutex_unlock(&zdev->state_lock); zpci_zdev_put(zdev); } @@ -326,6 +329,10 @@ static void __zpci_event_availability(struct zpci_ccdf_avail *ccdf) zpci_dbg(3, "avl fid:%x, fh:%x, pec:%x\n", ccdf->fid, ccdf->fh, ccdf->pec); + + if (existing_zdev) + mutex_lock(&zdev->state_lock); + switch (ccdf->pec) { case 0x0301: /* Reserved|Standby -> Configured */ if (!zdev) { @@ -348,7 +355,7 @@ static void __zpci_event_availability(struct zpci_ccdf_avail *ccdf) break; case 0x0303: /* Deconfiguration requested */ if (zdev) { - /* The event may have been queued before we confirgured + /* The event may have been queued before we configured * the device. */ if (zdev->state != ZPCI_FN_STATE_CONFIGURED) @@ -359,7 +366,7 @@ static void __zpci_event_availability(struct zpci_ccdf_avail *ccdf) break; case 0x0304: /* Configured -> Standby|Reserved */ if (zdev) { - /* The event may have been queued before we confirgured + /* The event may have been queued before we configured * the device.: */ if (zdev->state == ZPCI_FN_STATE_CONFIGURED) @@ -383,8 +390,10 @@ static void __zpci_event_availability(struct zpci_ccdf_avail *ccdf) default: break; } - if (existing_zdev) + if (existing_zdev) { + mutex_unlock(&zdev->state_lock); zpci_zdev_put(zdev); + } } void zpci_event_availability(void *data) diff --git a/arch/s390/pci/pci_sysfs.c b/arch/s390/pci/pci_sysfs.c index 8a7abac51816..a0b872b74fe3 100644 --- a/arch/s390/pci/pci_sysfs.c +++ b/arch/s390/pci/pci_sysfs.c @@ -49,6 +49,39 @@ static ssize_t mio_enabled_show(struct device *dev, } static DEVICE_ATTR_RO(mio_enabled); +static int _do_recover(struct pci_dev *pdev, struct zpci_dev *zdev) +{ + u8 status; + int ret; + + pci_stop_and_remove_bus_device(pdev); + if (zdev_enabled(zdev)) { + ret = zpci_disable_device(zdev); + /* + * Due to a z/VM vs LPAR inconsistency in the error + * state the FH may indicate an enabled device but + * disable says the device is already disabled don't + * treat it as an error here. + */ + if (ret == -EINVAL) + ret = 0; + if (ret) + return ret; + } + + ret = zpci_enable_device(zdev); + if (ret) + return ret; + + if (zdev->dma_table) { + ret = zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma, + virt_to_phys(zdev->dma_table), &status); + if (ret) + zpci_disable_device(zdev); + } + return ret; +} + static ssize_t recover_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { @@ -56,7 +89,6 @@ static ssize_t recover_store(struct device *dev, struct device_attribute *attr, struct pci_dev *pdev = to_pci_dev(dev); struct zpci_dev *zdev = to_zpci(pdev); int ret = 0; - u8 status; /* Can't use device_remove_self() here as that would lead us to lock * the pci_rescan_remove_lock while holding the device' kernfs lock. @@ -70,6 +102,12 @@ static ssize_t recover_store(struct device *dev, struct device_attribute *attr, */ kn = sysfs_break_active_protection(&dev->kobj, &attr->attr); WARN_ON_ONCE(!kn); + + /* Device needs to be configured and state must not change */ + mutex_lock(&zdev->state_lock); + if (zdev->state != ZPCI_FN_STATE_CONFIGURED) + goto out; + /* device_remove_file() serializes concurrent calls ignoring all but * the first */ @@ -82,35 +120,13 @@ static ssize_t recover_store(struct device *dev, struct device_attribute *attr, */ pci_lock_rescan_remove(); if (pci_dev_is_added(pdev)) { - pci_stop_and_remove_bus_device(pdev); - if (zdev_enabled(zdev)) { - ret = zpci_disable_device(zdev); - /* - * Due to a z/VM vs LPAR inconsistency in the error - * state the FH may indicate an enabled device but - * disable says the device is already disabled don't - * treat it as an error here. - */ - if (ret == -EINVAL) - ret = 0; - if (ret) - goto out; - } - - ret = zpci_enable_device(zdev); - if (ret) - goto out; - - if (zdev->dma_table) { - ret = zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma, - virt_to_phys(zdev->dma_table), &status); - if (ret) - zpci_disable_device(zdev); - } + ret = _do_recover(pdev, zdev); } -out: pci_rescan_bus(zdev->zbus->bus); pci_unlock_rescan_remove(); + +out: + mutex_unlock(&zdev->state_lock); if (kn) sysfs_unbreak_active_protection(kn); return ret ? ret : count; diff --git a/arch/s390/tools/.gitignore b/arch/s390/tools/.gitignore index ea62f37b79ef..e6af51d9d183 100644 --- a/arch/s390/tools/.gitignore +++ b/arch/s390/tools/.gitignore @@ -1,3 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only gen_facilities gen_opcode_table +relocs diff --git a/arch/s390/tools/Makefile b/arch/s390/tools/Makefile index f9dd47ff9ac4..f2862364fb42 100644 --- a/arch/s390/tools/Makefile +++ b/arch/s390/tools/Makefile @@ -25,3 +25,8 @@ $(kapi)/facility-defs.h: $(obj)/gen_facilities FORCE $(kapi)/dis-defs.h: $(obj)/gen_opcode_table FORCE $(call filechk,dis-defs.h) + +hostprogs += relocs +PHONY += relocs +relocs: $(obj)/relocs + @: diff --git a/arch/s390/tools/relocs.c b/arch/s390/tools/relocs.c new file mode 100644 index 000000000000..30a732c808f3 --- /dev/null +++ b/arch/s390/tools/relocs.c @@ -0,0 +1,387 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <stdio.h> +#include <stdarg.h> +#include <stdlib.h> +#include <stdint.h> +#include <inttypes.h> +#include <string.h> +#include <errno.h> +#include <unistd.h> +#include <elf.h> +#include <byteswap.h> +#define USE_BSD +#include <endian.h> + +#define ELF_BITS 64 + +#define ELF_MACHINE EM_S390 +#define ELF_MACHINE_NAME "IBM S/390" +#define SHT_REL_TYPE SHT_RELA +#define Elf_Rel Elf64_Rela + +#define ELF_CLASS ELFCLASS64 +#define ELF_ENDIAN ELFDATA2MSB +#define ELF_R_SYM(val) ELF64_R_SYM(val) +#define ELF_R_TYPE(val) ELF64_R_TYPE(val) +#define ELF_ST_TYPE(o) ELF64_ST_TYPE(o) +#define ELF_ST_BIND(o) ELF64_ST_BIND(o) +#define ELF_ST_VISIBILITY(o) ELF64_ST_VISIBILITY(o) + +#define ElfW(type) _ElfW(ELF_BITS, type) +#define _ElfW(bits, type) __ElfW(bits, type) +#define __ElfW(bits, type) Elf##bits##_##type + +#define Elf_Addr ElfW(Addr) +#define Elf_Ehdr ElfW(Ehdr) +#define Elf_Phdr ElfW(Phdr) +#define Elf_Shdr ElfW(Shdr) +#define Elf_Sym ElfW(Sym) + +static Elf_Ehdr ehdr; +static unsigned long shnum; +static unsigned int shstrndx; + +struct relocs { + uint32_t *offset; + unsigned long count; + unsigned long size; +}; + +static struct relocs relocs64; +#define FMT PRIu64 + +struct section { + Elf_Shdr shdr; + struct section *link; + Elf_Rel *reltab; +}; + +static struct section *secs; + +#if BYTE_ORDER == LITTLE_ENDIAN +#define le16_to_cpu(val) (val) +#define le32_to_cpu(val) (val) +#define le64_to_cpu(val) (val) +#define be16_to_cpu(val) bswap_16(val) +#define be32_to_cpu(val) bswap_32(val) +#define be64_to_cpu(val) bswap_64(val) +#endif + +#if BYTE_ORDER == BIG_ENDIAN +#define le16_to_cpu(val) bswap_16(val) +#define le32_to_cpu(val) bswap_32(val) +#define le64_to_cpu(val) bswap_64(val) +#define be16_to_cpu(val) (val) +#define be32_to_cpu(val) (val) +#define be64_to_cpu(val) (val) +#endif + +static uint16_t elf16_to_cpu(uint16_t val) +{ + if (ehdr.e_ident[EI_DATA] == ELFDATA2LSB) + return le16_to_cpu(val); + else + return be16_to_cpu(val); +} + +static uint32_t elf32_to_cpu(uint32_t val) +{ + if (ehdr.e_ident[EI_DATA] == ELFDATA2LSB) + return le32_to_cpu(val); + else + return be32_to_cpu(val); +} + +#define elf_half_to_cpu(x) elf16_to_cpu(x) +#define elf_word_to_cpu(x) elf32_to_cpu(x) + +static uint64_t elf64_to_cpu(uint64_t val) +{ + return be64_to_cpu(val); +} + +#define elf_addr_to_cpu(x) elf64_to_cpu(x) +#define elf_off_to_cpu(x) elf64_to_cpu(x) +#define elf_xword_to_cpu(x) elf64_to_cpu(x) + +static void die(char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + vfprintf(stderr, fmt, ap); + va_end(ap); + exit(1); +} + +static void read_ehdr(FILE *fp) +{ + if (fread(&ehdr, sizeof(ehdr), 1, fp) != 1) + die("Cannot read ELF header: %s\n", strerror(errno)); + if (memcmp(ehdr.e_ident, ELFMAG, SELFMAG) != 0) + die("No ELF magic\n"); + if (ehdr.e_ident[EI_CLASS] != ELF_CLASS) + die("Not a %d bit executable\n", ELF_BITS); + if (ehdr.e_ident[EI_DATA] != ELF_ENDIAN) + die("ELF endian mismatch\n"); + if (ehdr.e_ident[EI_VERSION] != EV_CURRENT) + die("Unknown ELF version\n"); + + /* Convert the fields to native endian */ + ehdr.e_type = elf_half_to_cpu(ehdr.e_type); + ehdr.e_machine = elf_half_to_cpu(ehdr.e_machine); + ehdr.e_version = elf_word_to_cpu(ehdr.e_version); + ehdr.e_entry = elf_addr_to_cpu(ehdr.e_entry); + ehdr.e_phoff = elf_off_to_cpu(ehdr.e_phoff); + ehdr.e_shoff = elf_off_to_cpu(ehdr.e_shoff); + ehdr.e_flags = elf_word_to_cpu(ehdr.e_flags); + ehdr.e_ehsize = elf_half_to_cpu(ehdr.e_ehsize); + ehdr.e_phentsize = elf_half_to_cpu(ehdr.e_phentsize); + ehdr.e_phnum = elf_half_to_cpu(ehdr.e_phnum); + ehdr.e_shentsize = elf_half_to_cpu(ehdr.e_shentsize); + ehdr.e_shnum = elf_half_to_cpu(ehdr.e_shnum); + ehdr.e_shstrndx = elf_half_to_cpu(ehdr.e_shstrndx); + + shnum = ehdr.e_shnum; + shstrndx = ehdr.e_shstrndx; + + if ((ehdr.e_type != ET_EXEC) && (ehdr.e_type != ET_DYN)) + die("Unsupported ELF header type\n"); + if (ehdr.e_machine != ELF_MACHINE) + die("Not for %s\n", ELF_MACHINE_NAME); + if (ehdr.e_version != EV_CURRENT) + die("Unknown ELF version\n"); + if (ehdr.e_ehsize != sizeof(Elf_Ehdr)) + die("Bad Elf header size\n"); + if (ehdr.e_phentsize != sizeof(Elf_Phdr)) + die("Bad program header entry\n"); + if (ehdr.e_shentsize != sizeof(Elf_Shdr)) + die("Bad section header entry\n"); + + if (shnum == SHN_UNDEF || shstrndx == SHN_XINDEX) { + Elf_Shdr shdr; + + if (fseek(fp, ehdr.e_shoff, SEEK_SET) < 0) + die("Seek to %" FMT " failed: %s\n", ehdr.e_shoff, strerror(errno)); + + if (fread(&shdr, sizeof(shdr), 1, fp) != 1) + die("Cannot read initial ELF section header: %s\n", strerror(errno)); + + if (shnum == SHN_UNDEF) + shnum = elf_xword_to_cpu(shdr.sh_size); + + if (shstrndx == SHN_XINDEX) + shstrndx = elf_word_to_cpu(shdr.sh_link); + } + + if (shstrndx >= shnum) + die("String table index out of bounds\n"); +} + +static void read_shdrs(FILE *fp) +{ + Elf_Shdr shdr; + int i; + + secs = calloc(shnum, sizeof(struct section)); + if (!secs) + die("Unable to allocate %ld section headers\n", shnum); + + if (fseek(fp, ehdr.e_shoff, SEEK_SET) < 0) + die("Seek to %" FMT " failed: %s\n", ehdr.e_shoff, strerror(errno)); + + for (i = 0; i < shnum; i++) { + struct section *sec = &secs[i]; + + if (fread(&shdr, sizeof(shdr), 1, fp) != 1) { + die("Cannot read ELF section headers %d/%ld: %s\n", + i, shnum, strerror(errno)); + } + + sec->shdr.sh_name = elf_word_to_cpu(shdr.sh_name); + sec->shdr.sh_type = elf_word_to_cpu(shdr.sh_type); + sec->shdr.sh_flags = elf_xword_to_cpu(shdr.sh_flags); + sec->shdr.sh_addr = elf_addr_to_cpu(shdr.sh_addr); + sec->shdr.sh_offset = elf_off_to_cpu(shdr.sh_offset); + sec->shdr.sh_size = elf_xword_to_cpu(shdr.sh_size); + sec->shdr.sh_link = elf_word_to_cpu(shdr.sh_link); + sec->shdr.sh_info = elf_word_to_cpu(shdr.sh_info); + sec->shdr.sh_addralign = elf_xword_to_cpu(shdr.sh_addralign); + sec->shdr.sh_entsize = elf_xword_to_cpu(shdr.sh_entsize); + + if (sec->shdr.sh_link < shnum) + sec->link = &secs[sec->shdr.sh_link]; + } + +} + +static void read_relocs(FILE *fp) +{ + int i, j; + + for (i = 0; i < shnum; i++) { + struct section *sec = &secs[i]; + + if (sec->shdr.sh_type != SHT_REL_TYPE) + continue; + + sec->reltab = malloc(sec->shdr.sh_size); + if (!sec->reltab) + die("malloc of %" FMT " bytes for relocs failed\n", sec->shdr.sh_size); + + if (fseek(fp, sec->shdr.sh_offset, SEEK_SET) < 0) + die("Seek to %" FMT " failed: %s\n", sec->shdr.sh_offset, strerror(errno)); + + if (fread(sec->reltab, 1, sec->shdr.sh_size, fp) != sec->shdr.sh_size) + die("Cannot read symbol table: %s\n", strerror(errno)); + + for (j = 0; j < sec->shdr.sh_size / sizeof(Elf_Rel); j++) { + Elf_Rel *rel = &sec->reltab[j]; + + rel->r_offset = elf_addr_to_cpu(rel->r_offset); + rel->r_info = elf_xword_to_cpu(rel->r_info); +#if (SHT_REL_TYPE == SHT_RELA) + rel->r_addend = elf_xword_to_cpu(rel->r_addend); +#endif + } + } +} + +static void add_reloc(struct relocs *r, uint32_t offset) +{ + if (r->count == r->size) { + unsigned long newsize = r->size + 50000; + void *mem = realloc(r->offset, newsize * sizeof(r->offset[0])); + + if (!mem) + die("realloc of %ld entries for relocs failed\n", newsize); + + r->offset = mem; + r->size = newsize; + } + r->offset[r->count++] = offset; +} + +static int do_reloc(struct section *sec, Elf_Rel *rel) +{ + unsigned int r_type = ELF64_R_TYPE(rel->r_info); + ElfW(Addr) offset = rel->r_offset; + + switch (r_type) { + case R_390_NONE: + case R_390_PC32: + case R_390_PC64: + case R_390_PC16DBL: + case R_390_PC32DBL: + case R_390_PLT32DBL: + case R_390_GOTENT: + case R_390_GOTPCDBL: + case R_390_GOTOFF64: + break; + case R_390_64: + add_reloc(&relocs64, offset); + break; + default: + die("Unsupported relocation type: %d\n", r_type); + break; + } + + return 0; +} + +static void walk_relocs(void) +{ + int i; + + /* Walk through the relocations */ + for (i = 0; i < shnum; i++) { + struct section *sec_applies; + int j; + struct section *sec = &secs[i]; + + if (sec->shdr.sh_type != SHT_REL_TYPE) + continue; + + sec_applies = &secs[sec->shdr.sh_info]; + if (!(sec_applies->shdr.sh_flags & SHF_ALLOC)) + continue; + + for (j = 0; j < sec->shdr.sh_size / sizeof(Elf_Rel); j++) { + Elf_Rel *rel = &sec->reltab[j]; + + do_reloc(sec, rel); + } + } +} + +static int cmp_relocs(const void *va, const void *vb) +{ + const uint32_t *a, *b; + + a = va; b = vb; + return (*a == *b) ? 0 : (*a > *b) ? 1 : -1; +} + +static void sort_relocs(struct relocs *r) +{ + qsort(r->offset, r->count, sizeof(r->offset[0]), cmp_relocs); +} + +static int print_reloc(uint32_t v) +{ + return fprintf(stdout, "\t.long 0x%08"PRIx32"\n", v) > 0 ? 0 : -1; +} + +static void emit_relocs(void) +{ + int i; + + walk_relocs(); + sort_relocs(&relocs64); + + printf(".section \".vmlinux.relocs_64\",\"a\"\n"); + for (i = 0; i < relocs64.count; i++) + print_reloc(relocs64.offset[i]); +} + +static void process(FILE *fp) +{ + read_ehdr(fp); + read_shdrs(fp); + read_relocs(fp); + emit_relocs(); +} + +static void usage(void) +{ + die("relocs vmlinux\n"); +} + +int main(int argc, char **argv) +{ + unsigned char e_ident[EI_NIDENT]; + const char *fname; + FILE *fp; + + fname = NULL; + + if (argc != 2) + usage(); + + fname = argv[1]; + + fp = fopen(fname, "r"); + if (!fp) + die("Cannot open %s: %s\n", fname, strerror(errno)); + + if (fread(&e_ident, 1, EI_NIDENT, fp) != EI_NIDENT) + die("Cannot read %s: %s", fname, strerror(errno)); + + rewind(fp); + + process(fp); + + fclose(fp); + return 0; +} diff --git a/drivers/pci/hotplug/s390_pci_hpc.c b/drivers/pci/hotplug/s390_pci_hpc.c index a89b7de72dcf..7333b305f2a5 100644 --- a/drivers/pci/hotplug/s390_pci_hpc.c +++ b/drivers/pci/hotplug/s390_pci_hpc.c @@ -26,58 +26,79 @@ static int enable_slot(struct hotplug_slot *hotplug_slot) hotplug_slot); int rc; - if (zdev->state != ZPCI_FN_STATE_STANDBY) - return -EIO; + mutex_lock(&zdev->state_lock); + if (zdev->state != ZPCI_FN_STATE_STANDBY) { + rc = -EIO; + goto out; + } rc = sclp_pci_configure(zdev->fid); zpci_dbg(3, "conf fid:%x, rc:%d\n", zdev->fid, rc); if (rc) - return rc; + goto out; zdev->state = ZPCI_FN_STATE_CONFIGURED; - return zpci_scan_configured_device(zdev, zdev->fh); + rc = zpci_scan_configured_device(zdev, zdev->fh); +out: + mutex_unlock(&zdev->state_lock); + return rc; } static int disable_slot(struct hotplug_slot *hotplug_slot) { struct zpci_dev *zdev = container_of(hotplug_slot, struct zpci_dev, hotplug_slot); - struct pci_dev *pdev; + struct pci_dev *pdev = NULL; + int rc; - if (zdev->state != ZPCI_FN_STATE_CONFIGURED) - return -EIO; + mutex_lock(&zdev->state_lock); + if (zdev->state != ZPCI_FN_STATE_CONFIGURED) { + rc = -EIO; + goto out; + } pdev = pci_get_slot(zdev->zbus->bus, zdev->devfn); if (pdev && pci_num_vf(pdev)) { pci_dev_put(pdev); - return -EBUSY; + rc = -EBUSY; + goto out; } - pci_dev_put(pdev); - return zpci_deconfigure_device(zdev); + rc = zpci_deconfigure_device(zdev); +out: + mutex_unlock(&zdev->state_lock); + if (pdev) + pci_dev_put(pdev); + return rc; } static int reset_slot(struct hotplug_slot *hotplug_slot, bool probe) { struct zpci_dev *zdev = container_of(hotplug_slot, struct zpci_dev, hotplug_slot); + int rc = -EIO; - if (zdev->state != ZPCI_FN_STATE_CONFIGURED) - return -EIO; /* - * We can't take the zdev->lock as reset_slot may be called during - * probing and/or device removal which already happens under the - * zdev->lock. Instead the user should use the higher level - * pci_reset_function() or pci_bus_reset() which hold the PCI device - * lock preventing concurrent removal. If not using these functions - * holding the PCI device lock is required. + * If we can't get the zdev->state_lock the device state is + * currently undergoing a transition and we bail out - just + * the same as if the device's state is not configured at all. */ + if (!mutex_trylock(&zdev->state_lock)) + return rc; - /* As long as the function is configured we can reset */ - if (probe) - return 0; + /* We can reset only if the function is configured */ + if (zdev->state != ZPCI_FN_STATE_CONFIGURED) + goto out; + + if (probe) { + rc = 0; + goto out; + } - return zpci_hot_reset_device(zdev); + rc = zpci_hot_reset_device(zdev); +out: + mutex_unlock(&zdev->state_lock); + return rc; } static int get_power_status(struct hotplug_slot *hotplug_slot, u8 *value) diff --git a/drivers/s390/char/vmur.c b/drivers/s390/char/vmur.c index 82efdd20ad01..1d17a83569ce 100644 --- a/drivers/s390/char/vmur.c +++ b/drivers/s390/char/vmur.c @@ -195,7 +195,7 @@ static void free_chan_prog(struct ccw1 *cpa) struct ccw1 *ptr = cpa; while (ptr->cda) { - kfree((void *)(addr_t) ptr->cda); + kfree(phys_to_virt(ptr->cda)); ptr++; } kfree(cpa); @@ -237,7 +237,7 @@ static struct ccw1 *alloc_chan_prog(const char __user *ubuf, int rec_count, free_chan_prog(cpa); return ERR_PTR(-ENOMEM); } - cpa[i].cda = (u32)(addr_t) kbuf; + cpa[i].cda = (u32)virt_to_phys(kbuf); if (copy_from_user(kbuf, ubuf, reclen)) { free_chan_prog(cpa); return ERR_PTR(-EFAULT); diff --git a/drivers/s390/char/zcore.c b/drivers/s390/char/zcore.c index bc3be0330f1d..0969fa01df58 100644 --- a/drivers/s390/char/zcore.c +++ b/drivers/s390/char/zcore.c @@ -29,7 +29,6 @@ #include <asm/irqflags.h> #include <asm/checksum.h> #include <asm/os_info.h> -#include <asm/switch_to.h> #include <asm/maccess.h> #include "sclp.h" diff --git a/drivers/s390/cio/ccwgroup.c b/drivers/s390/cio/ccwgroup.c index aa3292e57e38..6eb8bcd948dc 100644 --- a/drivers/s390/cio/ccwgroup.c +++ b/drivers/s390/cio/ccwgroup.c @@ -31,7 +31,7 @@ * to devices that use multiple subchannels. */ -static struct bus_type ccwgroup_bus_type; +static const struct bus_type ccwgroup_bus_type; static void __ccwgroup_remove_symlinks(struct ccwgroup_device *gdev) { @@ -465,7 +465,7 @@ static void ccwgroup_shutdown(struct device *dev) gdrv->shutdown(gdev); } -static struct bus_type ccwgroup_bus_type = { +static const struct bus_type ccwgroup_bus_type = { .name = "ccwgroup", .dev_groups = ccwgroup_dev_groups, .remove = ccwgroup_remove, diff --git a/drivers/s390/cio/chsc.c b/drivers/s390/cio/chsc.c index 64ed55c3aed6..3d88899dff7c 100644 --- a/drivers/s390/cio/chsc.c +++ b/drivers/s390/cio/chsc.c @@ -1091,8 +1091,8 @@ int __init chsc_init(void) { int ret; - sei_page = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA); - chsc_page = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA); + sei_page = (void *)get_zeroed_page(GFP_KERNEL); + chsc_page = (void *)get_zeroed_page(GFP_KERNEL); if (!sei_page || !chsc_page) { ret = -ENOMEM; goto out_err; diff --git a/drivers/s390/cio/chsc_sch.c b/drivers/s390/cio/chsc_sch.c index 902237d0baef..e6c800653f98 100644 --- a/drivers/s390/cio/chsc_sch.c +++ b/drivers/s390/cio/chsc_sch.c @@ -293,7 +293,7 @@ static int chsc_ioctl_start(void __user *user_area) if (!css_general_characteristics.dynio) /* It makes no sense to try. */ return -EOPNOTSUPP; - chsc_area = (void *)get_zeroed_page(GFP_DMA | GFP_KERNEL); + chsc_area = (void *)get_zeroed_page(GFP_KERNEL); if (!chsc_area) return -ENOMEM; request = kzalloc(sizeof(*request), GFP_KERNEL); @@ -341,7 +341,7 @@ static int chsc_ioctl_on_close_set(void __user *user_area) ret = -ENOMEM; goto out_unlock; } - on_close_chsc_area = (void *)get_zeroed_page(GFP_DMA | GFP_KERNEL); + on_close_chsc_area = (void *)get_zeroed_page(GFP_KERNEL); if (!on_close_chsc_area) { ret = -ENOMEM; goto out_free_request; @@ -393,7 +393,7 @@ static int chsc_ioctl_start_sync(void __user *user_area) struct chsc_sync_area *chsc_area; int ret, ccode; - chsc_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA); + chsc_area = (void *)get_zeroed_page(GFP_KERNEL); if (!chsc_area) return -ENOMEM; if (copy_from_user(chsc_area, user_area, PAGE_SIZE)) { @@ -439,7 +439,7 @@ static int chsc_ioctl_info_channel_path(void __user *user_cd) u8 data[PAGE_SIZE - 20]; } __attribute__ ((packed)) *scpcd_area; - scpcd_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA); + scpcd_area = (void *)get_zeroed_page(GFP_KERNEL); if (!scpcd_area) return -ENOMEM; cd = kzalloc(sizeof(*cd), GFP_KERNEL); @@ -501,7 +501,7 @@ static int chsc_ioctl_info_cu(void __user *user_cd) u8 data[PAGE_SIZE - 20]; } __attribute__ ((packed)) *scucd_area; - scucd_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA); + scucd_area = (void *)get_zeroed_page(GFP_KERNEL); if (!scucd_area) return -ENOMEM; cd = kzalloc(sizeof(*cd), GFP_KERNEL); @@ -564,7 +564,7 @@ static int chsc_ioctl_info_sch_cu(void __user *user_cud) u8 data[PAGE_SIZE - 20]; } __attribute__ ((packed)) *sscud_area; - sscud_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA); + sscud_area = (void *)get_zeroed_page(GFP_KERNEL); if (!sscud_area) return -ENOMEM; cud = kzalloc(sizeof(*cud), GFP_KERNEL); @@ -626,7 +626,7 @@ static int chsc_ioctl_conf_info(void __user *user_ci) u8 data[PAGE_SIZE - 20]; } __attribute__ ((packed)) *sci_area; - sci_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA); + sci_area = (void *)get_zeroed_page(GFP_KERNEL); if (!sci_area) return -ENOMEM; ci = kzalloc(sizeof(*ci), GFP_KERNEL); @@ -697,7 +697,7 @@ static int chsc_ioctl_conf_comp_list(void __user *user_ccl) u32 res; } __attribute__ ((packed)) *cssids_parm; - sccl_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA); + sccl_area = (void *)get_zeroed_page(GFP_KERNEL); if (!sccl_area) return -ENOMEM; ccl = kzalloc(sizeof(*ccl), GFP_KERNEL); @@ -757,7 +757,7 @@ static int chsc_ioctl_chpd(void __user *user_chpd) int ret; chpd = kzalloc(sizeof(*chpd), GFP_KERNEL); - scpd_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA); + scpd_area = (void *)get_zeroed_page(GFP_KERNEL); if (!scpd_area || !chpd) { ret = -ENOMEM; goto out_free; @@ -797,7 +797,7 @@ static int chsc_ioctl_dcal(void __user *user_dcal) u8 data[PAGE_SIZE - 36]; } __attribute__ ((packed)) *sdcal_area; - sdcal_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA); + sdcal_area = (void *)get_zeroed_page(GFP_KERNEL); if (!sdcal_area) return -ENOMEM; dcal = kzalloc(sizeof(*dcal), GFP_KERNEL); diff --git a/drivers/s390/cio/cmf.c b/drivers/s390/cio/cmf.c index 5584aa46c94e..f80dc18e2a76 100644 --- a/drivers/s390/cio/cmf.c +++ b/drivers/s390/cio/cmf.c @@ -169,7 +169,8 @@ static inline void cmf_activate(void *area, unsigned int onoff) " lgr 2,%[mbo]\n" " schm\n" : - : [r1] "d" ((unsigned long)onoff), [mbo] "d" (area) + : [r1] "d" ((unsigned long)onoff), + [mbo] "d" (virt_to_phys(area)) : "1", "2"); } @@ -501,8 +502,7 @@ static int alloc_cmb(struct ccw_device *cdev) WARN_ON(!list_empty(&cmb_area.list)); spin_unlock(&cmb_area.lock); - mem = (void*)__get_free_pages(GFP_KERNEL | GFP_DMA, - get_order(size)); + mem = (void *)__get_free_pages(GFP_KERNEL, get_order(size)); spin_lock(&cmb_area.lock); if (cmb_area.mem) { diff --git a/drivers/s390/cio/css.c b/drivers/s390/cio/css.c index 28a88ed2c3aa..094431a62ad5 100644 --- a/drivers/s390/cio/css.c +++ b/drivers/s390/cio/css.c @@ -39,7 +39,7 @@ int max_ssid; #define MAX_CSS_IDX 0 struct channel_subsystem *channel_subsystems[MAX_CSS_IDX + 1]; -static struct bus_type css_bus_type; +static const struct bus_type css_bus_type; int for_each_subchannel(int(*fn)(struct subchannel_id, void *), void *data) @@ -1409,7 +1409,7 @@ static int css_uevent(const struct device *dev, struct kobj_uevent_env *env) return ret; } -static struct bus_type css_bus_type = { +static const struct bus_type css_bus_type = { .name = "css", .match = css_bus_match, .probe = css_probe, diff --git a/drivers/s390/cio/device.c b/drivers/s390/cio/device.c index 0cfb179e1bcb..f95d12345d98 100644 --- a/drivers/s390/cio/device.c +++ b/drivers/s390/cio/device.c @@ -49,7 +49,7 @@ static const unsigned long recovery_delay[] = { 3, 30, 300 }; static atomic_t ccw_device_init_count = ATOMIC_INIT(0); static DECLARE_WAIT_QUEUE_HEAD(ccw_device_init_wq); -static struct bus_type ccw_bus_type; +static const struct bus_type ccw_bus_type; /******************* bus type handling ***********************/ @@ -1776,7 +1776,7 @@ static void ccw_device_shutdown(struct device *dev) __disable_cmf(cdev); } -static struct bus_type ccw_bus_type = { +static const struct bus_type ccw_bus_type = { .name = "ccw", .match = ccw_bus_match, .uevent = ccw_uevent, diff --git a/drivers/s390/cio/scm.c b/drivers/s390/cio/scm.c index 6b21ba68c1fe..c7894d61306d 100644 --- a/drivers/s390/cio/scm.c +++ b/drivers/s390/cio/scm.c @@ -42,7 +42,7 @@ static int scmdev_uevent(const struct device *dev, struct kobj_uevent_env *env) return add_uevent_var(env, "MODALIAS=scm:scmdev"); } -static struct bus_type scm_bus_type = { +static const struct bus_type scm_bus_type = { .name = "scm", .probe = scmdev_probe, .remove = scmdev_remove, @@ -228,7 +228,7 @@ int scm_update_information(void) size_t num; int ret; - scm_info = (void *)__get_free_page(GFP_KERNEL | GFP_DMA); + scm_info = (void *)__get_free_page(GFP_KERNEL); if (!scm_info) return -ENOMEM; diff --git a/drivers/s390/crypto/ap_bus.c b/drivers/s390/crypto/ap_bus.c index f46dd6abacd7..cce0bafd4c92 100644 --- a/drivers/s390/crypto/ap_bus.c +++ b/drivers/s390/crypto/ap_bus.c @@ -38,6 +38,7 @@ #include <linux/debugfs.h> #include <linux/ctype.h> #include <linux/module.h> +#include <asm/uv.h> #include "ap_bus.h" #include "ap_debug.h" @@ -83,14 +84,11 @@ EXPORT_SYMBOL(ap_perms); DEFINE_MUTEX(ap_perms_mutex); EXPORT_SYMBOL(ap_perms_mutex); -/* # of bus scans since init */ -static atomic64_t ap_scan_bus_count; - /* # of bindings complete since init */ static atomic64_t ap_bindings_complete_count = ATOMIC64_INIT(0); -/* completion for initial APQN bindings complete */ -static DECLARE_COMPLETION(ap_init_apqn_bindings_complete); +/* completion for APQN bindings complete */ +static DECLARE_COMPLETION(ap_apqn_bindings_complete); static struct ap_config_info *ap_qci_info; static struct ap_config_info *ap_qci_info_old; @@ -101,12 +99,16 @@ static struct ap_config_info *ap_qci_info_old; debug_info_t *ap_dbf_info; /* - * Workqueue timer for bus rescan. + * AP bus rescan related things. */ -static struct timer_list ap_config_timer; -static int ap_config_time = AP_CONFIG_TIME; -static void ap_scan_bus(struct work_struct *); -static DECLARE_WORK(ap_scan_work, ap_scan_bus); +static bool ap_scan_bus(void); +static bool ap_scan_bus_result; /* result of last ap_scan_bus() */ +static DEFINE_MUTEX(ap_scan_bus_mutex); /* mutex ap_scan_bus() invocations */ +static atomic64_t ap_scan_bus_count; /* counter ap_scan_bus() invocations */ +static int ap_scan_bus_time = AP_CONFIG_TIME; +static struct timer_list ap_scan_bus_timer; +static void ap_scan_bus_wq_callback(struct work_struct *); +static DECLARE_WORK(ap_scan_bus_work, ap_scan_bus_wq_callback); /* * Tasklet & timer for AP request polling and interrupts @@ -135,7 +137,7 @@ static int ap_max_domain_id = 15; /* Maximum adapter id, if not given via qci */ static int ap_max_adapter_id = 63; -static struct bus_type ap_bus_type; +static const struct bus_type ap_bus_type; /* Adapter interrupt definitions */ static void ap_interrupt_handler(struct airq_struct *airq, @@ -753,7 +755,7 @@ static void ap_calc_bound_apqns(unsigned int *apqns, unsigned int *bound) } /* - * After initial ap bus scan do check if all existing APQNs are + * After ap bus scan do check if all existing APQNs are * bound to device drivers. */ static void ap_check_bindings_complete(void) @@ -763,9 +765,9 @@ static void ap_check_bindings_complete(void) if (atomic64_read(&ap_scan_bus_count) >= 1) { ap_calc_bound_apqns(&apqns, &bound); if (bound == apqns) { - if (!completion_done(&ap_init_apqn_bindings_complete)) { - complete_all(&ap_init_apqn_bindings_complete); - AP_DBF_INFO("%s complete\n", __func__); + if (!completion_done(&ap_apqn_bindings_complete)) { + complete_all(&ap_apqn_bindings_complete); + pr_debug("%s all apqn bindings complete\n", __func__); } ap_send_bindings_complete_uevent(); } @@ -782,27 +784,29 @@ static void ap_check_bindings_complete(void) * -ETIME is returned. On failures negative return values are * returned to the caller. */ -int ap_wait_init_apqn_bindings_complete(unsigned long timeout) +int ap_wait_apqn_bindings_complete(unsigned long timeout) { + int rc = 0; long l; - if (completion_done(&ap_init_apqn_bindings_complete)) + if (completion_done(&ap_apqn_bindings_complete)) return 0; if (timeout) l = wait_for_completion_interruptible_timeout( - &ap_init_apqn_bindings_complete, timeout); + &ap_apqn_bindings_complete, timeout); else l = wait_for_completion_interruptible( - &ap_init_apqn_bindings_complete); + &ap_apqn_bindings_complete); if (l < 0) - return l == -ERESTARTSYS ? -EINTR : l; + rc = l == -ERESTARTSYS ? -EINTR : l; else if (l == 0 && timeout) - return -ETIME; + rc = -ETIME; - return 0; + pr_debug("%s rc=%d\n", __func__, rc); + return rc; } -EXPORT_SYMBOL(ap_wait_init_apqn_bindings_complete); +EXPORT_SYMBOL(ap_wait_apqn_bindings_complete); static int __ap_queue_devices_with_id_unregister(struct device *dev, void *data) { @@ -826,8 +830,8 @@ static int __ap_revise_reserved(struct device *dev, void *dummy) drvres = to_ap_drv(dev->driver)->flags & AP_DRIVER_FLAG_DEFAULT; if (!!devres != !!drvres) { - AP_DBF_DBG("%s reprobing queue=%02x.%04x\n", - __func__, card, queue); + pr_debug("%s reprobing queue=%02x.%04x\n", + __func__, card, queue); rc = device_reprobe(dev); if (rc) AP_DBF_WARN("%s reprobing queue=%02x.%04x failed\n", @@ -939,8 +943,6 @@ static int ap_device_probe(struct device *dev) if (is_queue_dev(dev)) hash_del(&to_ap_queue(dev)->hnode); spin_unlock_bh(&ap_queues_lock); - } else { - ap_check_bindings_complete(); } out: @@ -1012,16 +1014,47 @@ void ap_driver_unregister(struct ap_driver *ap_drv) } EXPORT_SYMBOL(ap_driver_unregister); -void ap_bus_force_rescan(void) +/* + * Enforce a synchronous AP bus rescan. + * Returns true if the bus scan finds a change in the AP configuration + * and AP devices have been added or deleted when this function returns. + */ +bool ap_bus_force_rescan(void) { + unsigned long scan_counter = atomic64_read(&ap_scan_bus_count); + bool rc = false; + + pr_debug(">%s scan counter=%lu\n", __func__, scan_counter); + /* Only trigger AP bus scans after the initial scan is done */ - if (atomic64_read(&ap_scan_bus_count) <= 0) - return; + if (scan_counter <= 0) + goto out; + + /* Try to acquire the AP scan bus mutex */ + if (mutex_trylock(&ap_scan_bus_mutex)) { + /* mutex acquired, run the AP bus scan */ + ap_scan_bus_result = ap_scan_bus(); + rc = ap_scan_bus_result; + mutex_unlock(&ap_scan_bus_mutex); + goto out; + } + + /* + * Mutex acquire failed. So there is currently another task + * already running the AP bus scan. Then let's simple wait + * for the lock which means the other task has finished and + * stored the result in ap_scan_bus_result. + */ + if (mutex_lock_interruptible(&ap_scan_bus_mutex)) { + /* some error occurred, ignore and go out */ + goto out; + } + rc = ap_scan_bus_result; + mutex_unlock(&ap_scan_bus_mutex); - /* processing a asynchronous bus rescan */ - del_timer(&ap_config_timer); - queue_work(system_long_wq, &ap_scan_work); - flush_work(&ap_scan_work); +out: + pr_debug("%s rc=%d\n", __func__, rc); + return rc; } EXPORT_SYMBOL(ap_bus_force_rescan); @@ -1030,7 +1063,7 @@ EXPORT_SYMBOL(ap_bus_force_rescan); */ void ap_bus_cfg_chg(void) { - AP_DBF_DBG("%s config change, forcing bus rescan\n", __func__); + pr_debug("%s config change, forcing bus rescan\n", __func__); ap_bus_force_rescan(); } @@ -1250,7 +1283,7 @@ static BUS_ATTR_RO(ap_interrupts); static ssize_t config_time_show(const struct bus_type *bus, char *buf) { - return sysfs_emit(buf, "%d\n", ap_config_time); + return sysfs_emit(buf, "%d\n", ap_scan_bus_time); } static ssize_t config_time_store(const struct bus_type *bus, @@ -1260,8 +1293,8 @@ static ssize_t config_time_store(const struct bus_type *bus, if (sscanf(buf, "%d\n", &time) != 1 || time < 5 || time > 120) return -EINVAL; - ap_config_time = time; - mod_timer(&ap_config_timer, jiffies + ap_config_time * HZ); + ap_scan_bus_time = time; + mod_timer(&ap_scan_bus_timer, jiffies + ap_scan_bus_time * HZ); return count; } @@ -1603,7 +1636,7 @@ static struct attribute *ap_bus_attrs[] = { }; ATTRIBUTE_GROUPS(ap_bus); -static struct bus_type ap_bus_type = { +static const struct bus_type ap_bus_type = { .name = "ap", .bus_groups = ap_bus_groups, .match = &ap_bus_match, @@ -1888,8 +1921,8 @@ static inline void ap_scan_domains(struct ap_card *ac) aq->last_err_rc = AP_RESPONSE_CHECKSTOPPED; } spin_unlock_bh(&aq->lock); - AP_DBF_DBG("%s(%d,%d) queue dev checkstop on\n", - __func__, ac->id, dom); + pr_debug("%s(%d,%d) queue dev checkstop on\n", + __func__, ac->id, dom); /* 'receive' pending messages with -EAGAIN */ ap_flush_queue(aq); goto put_dev_and_continue; @@ -1899,8 +1932,8 @@ static inline void ap_scan_domains(struct ap_card *ac) if (aq->dev_state > AP_DEV_STATE_UNINITIATED) _ap_queue_init_state(aq); spin_unlock_bh(&aq->lock); - AP_DBF_DBG("%s(%d,%d) queue dev checkstop off\n", - __func__, ac->id, dom); + pr_debug("%s(%d,%d) queue dev checkstop off\n", + __func__, ac->id, dom); goto put_dev_and_continue; } /* config state change */ @@ -1912,8 +1945,8 @@ static inline void ap_scan_domains(struct ap_card *ac) aq->last_err_rc = AP_RESPONSE_DECONFIGURED; } spin_unlock_bh(&aq->lock); - AP_DBF_DBG("%s(%d,%d) queue dev config off\n", - __func__, ac->id, dom); + pr_debug("%s(%d,%d) queue dev config off\n", + __func__, ac->id, dom); ap_send_config_uevent(&aq->ap_dev, aq->config); /* 'receive' pending messages with -EAGAIN */ ap_flush_queue(aq); @@ -1924,8 +1957,8 @@ static inline void ap_scan_domains(struct ap_card *ac) if (aq->dev_state > AP_DEV_STATE_UNINITIATED) _ap_queue_init_state(aq); spin_unlock_bh(&aq->lock); - AP_DBF_DBG("%s(%d,%d) queue dev config on\n", - __func__, ac->id, dom); + pr_debug("%s(%d,%d) queue dev config on\n", + __func__, ac->id, dom); ap_send_config_uevent(&aq->ap_dev, aq->config); goto put_dev_and_continue; } @@ -1997,8 +2030,8 @@ static inline void ap_scan_adapter(int ap) ap_scan_rm_card_dev_and_queue_devs(ac); put_device(dev); } else { - AP_DBF_DBG("%s(%d) no type info (no APQN found), ignored\n", - __func__, ap); + pr_debug("%s(%d) no type info (no APQN found), ignored\n", + __func__, ap); } return; } @@ -2010,8 +2043,8 @@ static inline void ap_scan_adapter(int ap) ap_scan_rm_card_dev_and_queue_devs(ac); put_device(dev); } else { - AP_DBF_DBG("%s(%d) no valid type (0) info, ignored\n", - __func__, ap); + pr_debug("%s(%d) no valid type (0) info, ignored\n", + __func__, ap); } return; } @@ -2135,23 +2168,80 @@ static bool ap_get_configuration(void) sizeof(struct ap_config_info)) != 0; } +/* + * ap_config_has_new_aps - Check current against old qci info if + * new adapters have appeared. Returns true if at least one new + * adapter in the apm mask is showing up. Existing adapters or + * receding adapters are not counted. + */ +static bool ap_config_has_new_aps(void) +{ + + unsigned long m[BITS_TO_LONGS(AP_DEVICES)]; + + if (!ap_qci_info) + return false; + + bitmap_andnot(m, (unsigned long *)ap_qci_info->apm, + (unsigned long *)ap_qci_info_old->apm, AP_DEVICES); + if (!bitmap_empty(m, AP_DEVICES)) + return true; + + return false; +} + +/* + * ap_config_has_new_doms - Check current against old qci info if + * new (usage) domains have appeared. Returns true if at least one + * new domain in the aqm mask is showing up. Existing domains or + * receding domains are not counted. + */ +static bool ap_config_has_new_doms(void) +{ + unsigned long m[BITS_TO_LONGS(AP_DOMAINS)]; + + if (!ap_qci_info) + return false; + + bitmap_andnot(m, (unsigned long *)ap_qci_info->aqm, + (unsigned long *)ap_qci_info_old->aqm, AP_DOMAINS); + if (!bitmap_empty(m, AP_DOMAINS)) + return true; + + return false; +} + /** * ap_scan_bus(): Scan the AP bus for new devices - * Runs periodically, workqueue timer (ap_config_time) - * @unused: Unused pointer. + * Always run under mutex ap_scan_bus_mutex protection + * which needs to get locked/unlocked by the caller! + * Returns true if any config change has been detected + * during the scan, otherwise false. */ -static void ap_scan_bus(struct work_struct *unused) +static bool ap_scan_bus(void) { - int ap, config_changed = 0; + bool config_changed; + int ap; + + pr_debug(">%s\n", __func__); - /* config change notify */ + /* (re-)fetch configuration via QCI */ config_changed = ap_get_configuration(); - if (config_changed) + if (config_changed) { + if (ap_config_has_new_aps() || ap_config_has_new_doms()) { + /* + * Appearance of new adapters and/or domains need to + * build new ap devices which need to get bound to an + * device driver. Thus reset the APQN bindings complete + * completion. + */ + reinit_completion(&ap_apqn_bindings_complete); + } + /* post a config change notify */ notify_config_changed(); + } ap_select_domain(); - AP_DBF_DBG("%s running\n", __func__); - /* loop over all possible adapters */ for (ap = 0; ap <= ap_max_adapter_id; ap++) ap_scan_adapter(ap); @@ -2174,23 +2264,56 @@ static void ap_scan_bus(struct work_struct *unused) } if (atomic64_inc_return(&ap_scan_bus_count) == 1) { - AP_DBF_DBG("%s init scan complete\n", __func__); + pr_debug("%s init scan complete\n", __func__); ap_send_init_scan_done_uevent(); - ap_check_bindings_complete(); } - mod_timer(&ap_config_timer, jiffies + ap_config_time * HZ); + ap_check_bindings_complete(); + + mod_timer(&ap_scan_bus_timer, jiffies + ap_scan_bus_time * HZ); + + pr_debug("<%s config_changed=%d\n", __func__, config_changed); + + return config_changed; } -static void ap_config_timeout(struct timer_list *unused) +/* + * Callback for the ap_scan_bus_timer + * Runs periodically, workqueue timer (ap_scan_bus_time) + */ +static void ap_scan_bus_timer_callback(struct timer_list *unused) { - queue_work(system_long_wq, &ap_scan_work); + /* + * schedule work into the system long wq which when + * the work is finally executed, calls the AP bus scan. + */ + queue_work(system_long_wq, &ap_scan_bus_work); +} + +/* + * Callback for the ap_scan_bus_work + */ +static void ap_scan_bus_wq_callback(struct work_struct *unused) +{ + /* + * Try to invoke an ap_scan_bus(). If the mutex acquisition + * fails there is currently another task already running the + * AP scan bus and there is no need to wait and re-trigger the + * scan again. Please note at the end of the scan bus function + * the AP scan bus timer is re-armed which triggers then the + * ap_scan_bus_timer_callback which enqueues a work into the + * system_long_wq which invokes this function here again. + */ + if (mutex_trylock(&ap_scan_bus_mutex)) { + ap_scan_bus_result = ap_scan_bus(); + mutex_unlock(&ap_scan_bus_mutex); + } } static int __init ap_debug_init(void) { ap_dbf_info = debug_register("ap", 2, 1, - DBF_MAX_SPRINTF_ARGS * sizeof(long)); + AP_DBF_MAX_SPRINTF_ARGS * sizeof(long)); debug_register_view(ap_dbf_info, &debug_sprintf_view); debug_set_level(ap_dbf_info, DBF_ERR); @@ -2274,7 +2397,7 @@ static int __init ap_module_init(void) ap_root_device->bus = &ap_bus_type; /* Setup the AP bus rescan timer. */ - timer_setup(&ap_config_timer, ap_config_timeout, 0); + timer_setup(&ap_scan_bus_timer, ap_scan_bus_timer_callback, 0); /* * Setup the high resolution poll timer. @@ -2292,7 +2415,7 @@ static int __init ap_module_init(void) goto out_work; } - queue_work(system_long_wq, &ap_scan_work); + queue_work(system_long_wq, &ap_scan_bus_work); return 0; diff --git a/drivers/s390/crypto/ap_bus.h b/drivers/s390/crypto/ap_bus.h index 98814839ef30..59c7ed49aa02 100644 --- a/drivers/s390/crypto/ap_bus.h +++ b/drivers/s390/crypto/ap_bus.h @@ -266,7 +266,7 @@ int ap_sb_available(void); bool ap_is_se_guest(void); void ap_wait(enum ap_sm_wait wait); void ap_request_timeout(struct timer_list *t); -void ap_bus_force_rescan(void); +bool ap_bus_force_rescan(void); int ap_test_config_usage_domain(unsigned int domain); int ap_test_config_ctrl_domain(unsigned int domain); @@ -352,8 +352,12 @@ int ap_parse_mask_str(const char *str, * the return value is 0. If the timeout (in jiffies) hits instead * -ETIME is returned. On failures negative return values are * returned to the caller. + * It may be that the AP bus scan finds new devices. Then the + * condition that all APQNs are bound to their device drivers + * is reset to false and this call again blocks until either all + * APQNs are bound to a device driver or the timeout hits again. */ -int ap_wait_init_apqn_bindings_complete(unsigned long timeout); +int ap_wait_apqn_bindings_complete(unsigned long timeout); void ap_send_config_uevent(struct ap_device *ap_dev, bool cfg); void ap_send_online_uevent(struct ap_device *ap_dev, int online); diff --git a/drivers/s390/crypto/ap_debug.h b/drivers/s390/crypto/ap_debug.h index c083ce88a9a6..2f66271b8564 100644 --- a/drivers/s390/crypto/ap_debug.h +++ b/drivers/s390/crypto/ap_debug.h @@ -16,7 +16,7 @@ #define RC2ERR(rc) ((rc) ? DBF_ERR : DBF_INFO) #define RC2WARN(rc) ((rc) ? DBF_WARN : DBF_INFO) -#define DBF_MAX_SPRINTF_ARGS 6 +#define AP_DBF_MAX_SPRINTF_ARGS 6 #define AP_DBF(...) \ debug_sprintf_event(ap_dbf_info, ##__VA_ARGS__) @@ -26,8 +26,6 @@ debug_sprintf_event(ap_dbf_info, DBF_WARN, ##__VA_ARGS__) #define AP_DBF_INFO(...) \ debug_sprintf_event(ap_dbf_info, DBF_INFO, ##__VA_ARGS__) -#define AP_DBF_DBG(...) \ - debug_sprintf_event(ap_dbf_info, DBF_DEBUG, ##__VA_ARGS__) extern debug_info_t *ap_dbf_info; diff --git a/drivers/s390/crypto/ap_queue.c b/drivers/s390/crypto/ap_queue.c index 682595443145..6e4e8d324a6d 100644 --- a/drivers/s390/crypto/ap_queue.c +++ b/drivers/s390/crypto/ap_queue.c @@ -136,6 +136,8 @@ static struct ap_queue_status ap_sm_recv(struct ap_queue *aq) switch (status.response_code) { case AP_RESPONSE_NORMAL: + print_hex_dump_debug("aprpl: ", DUMP_PREFIX_ADDRESS, 16, 1, + aq->reply->msg, aq->reply->len, false); aq->queue_count = max_t(int, 0, aq->queue_count - 1); if (!status.queue_empty && !aq->queue_count) aq->queue_count++; @@ -169,6 +171,9 @@ static struct ap_queue_status ap_sm_recv(struct ap_queue *aq) aq->queue_count = 0; list_splice_init(&aq->pendingq, &aq->requestq); aq->requestq_count += aq->pendingq_count; + pr_debug("%s queue 0x%02x.%04x rescheduled %d reqs (new req %d)\n", + __func__, AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid), + aq->pendingq_count, aq->requestq_count); aq->pendingq_count = 0; break; default: @@ -243,6 +248,8 @@ static enum ap_sm_wait ap_sm_write(struct ap_queue *aq) /* Start the next request on the queue. */ ap_msg = list_entry(aq->requestq.next, struct ap_message, list); + print_hex_dump_debug("apreq: ", DUMP_PREFIX_ADDRESS, 16, 1, + ap_msg->msg, ap_msg->len, false); status = __ap_send(qid, ap_msg->psmid, ap_msg->msg, ap_msg->len, ap_msg->flags & AP_MSG_FLAG_SPECIAL); @@ -446,9 +453,9 @@ static enum ap_sm_wait ap_sm_assoc_wait(struct ap_queue *aq) case AP_BS_Q_USABLE: /* association is through */ aq->sm_state = AP_SM_STATE_IDLE; - AP_DBF_DBG("%s queue 0x%02x.%04x associated with %u\n", - __func__, AP_QID_CARD(aq->qid), - AP_QID_QUEUE(aq->qid), aq->assoc_idx); + pr_debug("%s queue 0x%02x.%04x associated with %u\n", + __func__, AP_QID_CARD(aq->qid), + AP_QID_QUEUE(aq->qid), aq->assoc_idx); return AP_SM_WAIT_NONE; case AP_BS_Q_USABLE_NO_SECURE_KEY: /* association still pending */ @@ -690,9 +697,9 @@ static ssize_t ap_functions_show(struct device *dev, status = ap_test_queue(aq->qid, 1, &hwinfo); if (status.response_code > AP_RESPONSE_BUSY) { - AP_DBF_DBG("%s RC 0x%02x on tapq(0x%02x.%04x)\n", - __func__, status.response_code, - AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid)); + pr_debug("%s RC 0x%02x on tapq(0x%02x.%04x)\n", + __func__, status.response_code, + AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid)); return -EIO; } @@ -846,9 +853,9 @@ static ssize_t se_bind_show(struct device *dev, status = ap_test_queue(aq->qid, 1, &hwinfo); if (status.response_code > AP_RESPONSE_BUSY) { - AP_DBF_DBG("%s RC 0x%02x on tapq(0x%02x.%04x)\n", - __func__, status.response_code, - AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid)); + pr_debug("%s RC 0x%02x on tapq(0x%02x.%04x)\n", + __func__, status.response_code, + AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid)); return -EIO; } @@ -974,9 +981,9 @@ static ssize_t se_associate_show(struct device *dev, status = ap_test_queue(aq->qid, 1, &hwinfo); if (status.response_code > AP_RESPONSE_BUSY) { - AP_DBF_DBG("%s RC 0x%02x on tapq(0x%02x.%04x)\n", - __func__, status.response_code, - AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid)); + pr_debug("%s RC 0x%02x on tapq(0x%02x.%04x)\n", + __func__, status.response_code, + AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid)); return -EIO; } diff --git a/drivers/s390/crypto/pkey_api.c b/drivers/s390/crypto/pkey_api.c index 6cfb6b2340c9..dccf664a3d95 100644 --- a/drivers/s390/crypto/pkey_api.c +++ b/drivers/s390/crypto/pkey_api.c @@ -42,24 +42,23 @@ MODULE_DESCRIPTION("s390 protected key interface"); * debug feature data and functions */ -static debug_info_t *debug_info; +static debug_info_t *pkey_dbf_info; -#define DEBUG_DBG(...) debug_sprintf_event(debug_info, 6, ##__VA_ARGS__) -#define DEBUG_INFO(...) debug_sprintf_event(debug_info, 5, ##__VA_ARGS__) -#define DEBUG_WARN(...) debug_sprintf_event(debug_info, 4, ##__VA_ARGS__) -#define DEBUG_ERR(...) debug_sprintf_event(debug_info, 3, ##__VA_ARGS__) +#define PKEY_DBF_INFO(...) debug_sprintf_event(pkey_dbf_info, 5, ##__VA_ARGS__) +#define PKEY_DBF_WARN(...) debug_sprintf_event(pkey_dbf_info, 4, ##__VA_ARGS__) +#define PKEY_DBF_ERR(...) debug_sprintf_event(pkey_dbf_info, 3, ##__VA_ARGS__) static void __init pkey_debug_init(void) { /* 5 arguments per dbf entry (including the format string ptr) */ - debug_info = debug_register("pkey", 1, 1, 5 * sizeof(long)); - debug_register_view(debug_info, &debug_sprintf_view); - debug_set_level(debug_info, 3); + pkey_dbf_info = debug_register("pkey", 1, 1, 5 * sizeof(long)); + debug_register_view(pkey_dbf_info, &debug_sprintf_view); + debug_set_level(pkey_dbf_info, 3); } static void __exit pkey_debug_exit(void) { - debug_unregister(debug_info); + debug_unregister(pkey_dbf_info); } /* inside view of a protected key token (only type 0x00 version 0x01) */ @@ -163,14 +162,14 @@ static int pkey_clr2protkey(u32 keytype, const u8 *clrkey, fc = CPACF_PCKMO_ENC_ECC_ED448_KEY; break; default: - DEBUG_ERR("%s unknown/unsupported keytype %u\n", - __func__, keytype); + PKEY_DBF_ERR("%s unknown/unsupported keytype %u\n", + __func__, keytype); return -EINVAL; } if (*protkeylen < keysize + AES_WK_VP_SIZE) { - DEBUG_ERR("%s prot key buffer size too small: %u < %d\n", - __func__, *protkeylen, keysize + AES_WK_VP_SIZE); + PKEY_DBF_ERR("%s prot key buffer size too small: %u < %d\n", + __func__, *protkeylen, keysize + AES_WK_VP_SIZE); return -EINVAL; } @@ -182,7 +181,7 @@ static int pkey_clr2protkey(u32 keytype, const u8 *clrkey, } /* check for the pckmo subfunction we need now */ if (!cpacf_test_func(&pckmo_functions, fc)) { - DEBUG_ERR("%s pckmo functions not available\n", __func__); + PKEY_DBF_ERR("%s pckmo functions not available\n", __func__); return -ENODEV; } @@ -244,7 +243,7 @@ static int pkey_skey2pkey(const u8 *key, u8 *protkey, } if (rc) - DEBUG_DBG("%s failed rc=%d\n", __func__, rc); + pr_debug("%s failed rc=%d\n", __func__, rc); return rc; } @@ -283,7 +282,7 @@ static int pkey_clr2ep11key(const u8 *clrkey, size_t clrkeylen, out: kfree(apqns); if (rc) - DEBUG_DBG("%s failed rc=%d\n", __func__, rc); + pr_debug("%s failed rc=%d\n", __func__, rc); return rc; } @@ -294,33 +293,36 @@ static int pkey_ep11key2pkey(const u8 *key, size_t keylen, u8 *protkey, u32 *protkeylen, u32 *protkeytype) { u32 nr_apqns, *apqns = NULL; + int i, j, rc = -ENODEV; u16 card, dom; - int i, rc; zcrypt_wait_api_operational(); - /* build a list of apqns suitable for this key */ - rc = ep11_findcard2(&apqns, &nr_apqns, 0xFFFF, 0xFFFF, - ZCRYPT_CEX7, - ap_is_se_guest() ? EP11_API_V6 : EP11_API_V4, - ep11_kb_wkvp(key, keylen)); - if (rc) - goto out; + /* try two times in case of failure */ + for (i = 0; i < 2 && rc; i++) { - /* go through the list of apqns and try to derive an pkey */ - for (rc = -ENODEV, i = 0; i < nr_apqns; i++) { - card = apqns[i] >> 16; - dom = apqns[i] & 0xFFFF; - rc = ep11_kblob2protkey(card, dom, key, keylen, - protkey, protkeylen, protkeytype); - if (rc == 0) - break; + /* build a list of apqns suitable for this key */ + rc = ep11_findcard2(&apqns, &nr_apqns, 0xFFFF, 0xFFFF, + ZCRYPT_CEX7, + ap_is_se_guest() ? EP11_API_V6 : EP11_API_V4, + ep11_kb_wkvp(key, keylen)); + if (rc) + continue; /* retry findcard on failure */ + + /* go through the list of apqns and try to derive an pkey */ + for (rc = -ENODEV, j = 0; j < nr_apqns && rc; j++) { + card = apqns[j] >> 16; + dom = apqns[j] & 0xFFFF; + rc = ep11_kblob2protkey(card, dom, key, keylen, + protkey, protkeylen, protkeytype); + } + + kfree(apqns); } -out: - kfree(apqns); if (rc) - DEBUG_DBG("%s failed rc=%d\n", __func__, rc); + pr_debug("%s failed rc=%d\n", __func__, rc); + return rc; } @@ -336,7 +338,7 @@ static int pkey_verifykey(const struct pkey_seckey *seckey, int rc; /* check the secure key for valid AES secure key */ - rc = cca_check_secaeskeytoken(debug_info, 3, (u8 *)seckey, 0); + rc = cca_check_secaeskeytoken(pkey_dbf_info, 3, (u8 *)seckey, 0); if (rc) goto out; if (pattributes) @@ -351,7 +353,7 @@ static int pkey_verifykey(const struct pkey_seckey *seckey, if (rc > 0) { /* key mkvp matches to old master key mkvp */ - DEBUG_DBG("%s secure key has old mkvp\n", __func__); + pr_debug("%s secure key has old mkvp\n", __func__); if (pattributes) *pattributes |= PKEY_VERIFY_ATTR_OLD_MKVP; rc = 0; @@ -363,7 +365,7 @@ static int pkey_verifykey(const struct pkey_seckey *seckey, *pdomain = domain; out: - DEBUG_DBG("%s rc=%d\n", __func__, rc); + pr_debug("%s rc=%d\n", __func__, rc); return rc; } @@ -379,8 +381,8 @@ static int pkey_genprotkey(u32 keytype, u8 *protkey, keysize = pkey_keytype_aes_to_size(keytype); if (!keysize) { - DEBUG_ERR("%s unknown/unsupported keytype %d\n", __func__, - keytype); + PKEY_DBF_ERR("%s unknown/unsupported keytype %d\n", __func__, + keytype); return -EINVAL; } @@ -428,13 +430,13 @@ static int pkey_verifyprotkey(const u8 *protkey, u32 protkeylen, fc = CPACF_KMC_PAES_256; break; default: - DEBUG_ERR("%s unknown/unsupported keytype %u\n", __func__, - protkeytype); + PKEY_DBF_ERR("%s unknown/unsupported keytype %u\n", __func__, + protkeytype); return -EINVAL; } if (protkeylen != pkeylen) { - DEBUG_ERR("%s invalid protected key size %u for keytype %u\n", - __func__, protkeylen, protkeytype); + PKEY_DBF_ERR("%s invalid protected key size %u for keytype %u\n", + __func__, protkeylen, protkeytype); return -EINVAL; } @@ -446,7 +448,7 @@ static int pkey_verifyprotkey(const u8 *protkey, u32 protkeylen, k = cpacf_kmc(fc | CPACF_ENCRYPT, ¶m, null_msg, dest_buf, sizeof(null_msg)); if (k != sizeof(null_msg)) { - DEBUG_ERR("%s protected key is not valid\n", __func__); + PKEY_DBF_ERR("%s protected key is not valid\n", __func__); return -EKEYREJECTED; } @@ -464,13 +466,13 @@ static int nonccatokaes2pkey(const struct clearkeytoken *t, keysize = pkey_keytype_aes_to_size(t->keytype); if (!keysize) { - DEBUG_ERR("%s unknown/unsupported keytype %u\n", - __func__, t->keytype); + PKEY_DBF_ERR("%s unknown/unsupported keytype %u\n", + __func__, t->keytype); return -EINVAL; } if (t->len != keysize) { - DEBUG_ERR("%s non clear key aes token: invalid key len %u\n", - __func__, t->len); + PKEY_DBF_ERR("%s non clear key aes token: invalid key len %u\n", + __func__, t->len); return -EINVAL; } @@ -505,7 +507,7 @@ try_via_ep11: goto out; failure: - DEBUG_ERR("%s unable to build protected key from clear", __func__); + PKEY_DBF_ERR("%s unable to build protected key from clear", __func__); out: kfree(tmpbuf); @@ -536,14 +538,14 @@ static int nonccatokecc2pkey(const struct clearkeytoken *t, keylen = 64; break; default: - DEBUG_ERR("%s unknown/unsupported keytype %u\n", - __func__, t->keytype); + PKEY_DBF_ERR("%s unknown/unsupported keytype %u\n", + __func__, t->keytype); return -EINVAL; } if (t->len != keylen) { - DEBUG_ERR("%s non clear key ecc token: invalid key len %u\n", - __func__, t->len); + PKEY_DBF_ERR("%s non clear key ecc token: invalid key len %u\n", + __func__, t->len); return -EINVAL; } @@ -551,8 +553,8 @@ static int nonccatokecc2pkey(const struct clearkeytoken *t, rc = pkey_clr2protkey(t->keytype, t->clearkey, protkey, protkeylen, protkeytype); if (rc) { - DEBUG_ERR("%s unable to build protected key from clear", - __func__); + PKEY_DBF_ERR("%s unable to build protected key from clear", + __func__); } return rc; @@ -604,15 +606,15 @@ static int pkey_nonccatok2pkey(const u8 *key, u32 keylen, protkeylen, protkeytype); break; default: - DEBUG_ERR("%s unknown/unsupported non cca clear key type %u\n", - __func__, t->keytype); + PKEY_DBF_ERR("%s unknown/unsupported non cca clear key type %u\n", + __func__, t->keytype); return -EINVAL; } break; } case TOKVER_EP11_AES: { /* check ep11 key for exportable as protected key */ - rc = ep11_check_aes_key(debug_info, 3, key, keylen, 1); + rc = ep11_check_aes_key(pkey_dbf_info, 3, key, keylen, 1); if (rc) goto out; rc = pkey_ep11key2pkey(key, keylen, @@ -621,15 +623,16 @@ static int pkey_nonccatok2pkey(const u8 *key, u32 keylen, } case TOKVER_EP11_AES_WITH_HEADER: /* check ep11 key with header for exportable as protected key */ - rc = ep11_check_aes_key_with_hdr(debug_info, 3, key, keylen, 1); + rc = ep11_check_aes_key_with_hdr(pkey_dbf_info, + 3, key, keylen, 1); if (rc) goto out; rc = pkey_ep11key2pkey(key, keylen, protkey, protkeylen, protkeytype); break; default: - DEBUG_ERR("%s unknown/unsupported non-CCA token version %d\n", - __func__, hdr->version); + PKEY_DBF_ERR("%s unknown/unsupported non-CCA token version %d\n", + __func__, hdr->version); } out: @@ -654,8 +657,8 @@ static int pkey_ccainttok2pkey(const u8 *key, u32 keylen, return -EINVAL; break; default: - DEBUG_ERR("%s unknown/unsupported CCA internal token version %d\n", - __func__, hdr->version); + PKEY_DBF_ERR("%s unknown/unsupported CCA internal token version %d\n", + __func__, hdr->version); return -EINVAL; } @@ -672,7 +675,7 @@ int pkey_keyblob2pkey(const u8 *key, u32 keylen, int rc; if (keylen < sizeof(struct keytoken_header)) { - DEBUG_ERR("%s invalid keylen %d\n", __func__, keylen); + PKEY_DBF_ERR("%s invalid keylen %d\n", __func__, keylen); return -EINVAL; } @@ -686,12 +689,12 @@ int pkey_keyblob2pkey(const u8 *key, u32 keylen, protkey, protkeylen, protkeytype); break; default: - DEBUG_ERR("%s unknown/unsupported blob type %d\n", - __func__, hdr->type); + PKEY_DBF_ERR("%s unknown/unsupported blob type %d\n", + __func__, hdr->type); return -EINVAL; } - DEBUG_DBG("%s rc=%d\n", __func__, rc); + pr_debug("%s rc=%d\n", __func__, rc); return rc; } EXPORT_SYMBOL(pkey_keyblob2pkey); @@ -839,7 +842,7 @@ static int pkey_verifykey2(const u8 *key, size_t keylen, hdr->version == TOKVER_CCA_AES) { struct secaeskeytoken *t = (struct secaeskeytoken *)key; - rc = cca_check_secaeskeytoken(debug_info, 3, key, 0); + rc = cca_check_secaeskeytoken(pkey_dbf_info, 3, key, 0); if (rc) goto out; if (ktype) @@ -869,7 +872,7 @@ static int pkey_verifykey2(const u8 *key, size_t keylen, hdr->version == TOKVER_CCA_VLSC) { struct cipherkeytoken *t = (struct cipherkeytoken *)key; - rc = cca_check_secaescipherkey(debug_info, 3, key, 0, 1); + rc = cca_check_secaescipherkey(pkey_dbf_info, 3, key, 0, 1); if (rc) goto out; if (ktype) @@ -907,7 +910,7 @@ static int pkey_verifykey2(const u8 *key, size_t keylen, struct ep11keyblob *kb = (struct ep11keyblob *)key; int api; - rc = ep11_check_aes_key(debug_info, 3, key, keylen, 1); + rc = ep11_check_aes_key(pkey_dbf_info, 3, key, keylen, 1); if (rc) goto out; if (ktype) @@ -933,8 +936,8 @@ static int pkey_verifykey2(const u8 *key, size_t keylen, struct ep11kblob_header *kh = (struct ep11kblob_header *)key; int api; - rc = ep11_check_aes_key_with_hdr(debug_info, 3, - key, keylen, 1); + rc = ep11_check_aes_key_with_hdr(pkey_dbf_info, + 3, key, keylen, 1); if (rc) goto out; if (ktype) @@ -981,25 +984,27 @@ static int pkey_keyblob2pkey2(const struct pkey_apqn *apqns, size_t nr_apqns, if (hdr->version == TOKVER_CCA_AES) { if (keylen != sizeof(struct secaeskeytoken)) return -EINVAL; - if (cca_check_secaeskeytoken(debug_info, 3, key, 0)) + if (cca_check_secaeskeytoken(pkey_dbf_info, 3, key, 0)) return -EINVAL; } else if (hdr->version == TOKVER_CCA_VLSC) { if (keylen < hdr->len || keylen > MAXCCAVLSCTOKENSIZE) return -EINVAL; - if (cca_check_secaescipherkey(debug_info, 3, key, 0, 1)) + if (cca_check_secaescipherkey(pkey_dbf_info, + 3, key, 0, 1)) return -EINVAL; } else { - DEBUG_ERR("%s unknown CCA internal token version %d\n", - __func__, hdr->version); + PKEY_DBF_ERR("%s unknown CCA internal token version %d\n", + __func__, hdr->version); return -EINVAL; } } else if (hdr->type == TOKTYPE_NON_CCA) { if (hdr->version == TOKVER_EP11_AES) { - if (ep11_check_aes_key(debug_info, 3, key, keylen, 1)) + if (ep11_check_aes_key(pkey_dbf_info, + 3, key, keylen, 1)) return -EINVAL; } else if (hdr->version == TOKVER_EP11_AES_WITH_HEADER) { - if (ep11_check_aes_key_with_hdr(debug_info, 3, - key, keylen, 1)) + if (ep11_check_aes_key_with_hdr(pkey_dbf_info, + 3, key, keylen, 1)) return -EINVAL; } else { return pkey_nonccatok2pkey(key, keylen, @@ -1007,8 +1012,8 @@ static int pkey_keyblob2pkey2(const struct pkey_apqn *apqns, size_t nr_apqns, protkeytype); } } else { - DEBUG_ERR("%s unknown/unsupported blob type %d\n", - __func__, hdr->type); + PKEY_DBF_ERR("%s unknown/unsupported blob type %d\n", + __func__, hdr->type); return -EINVAL; } @@ -1234,50 +1239,53 @@ static int pkey_keyblob2pkey3(const struct pkey_apqn *apqns, size_t nr_apqns, hdr->version == TOKVER_EP11_AES_WITH_HEADER && is_ep11_keyblob(key + sizeof(struct ep11kblob_header))) { /* EP11 AES key blob with header */ - if (ep11_check_aes_key_with_hdr(debug_info, 3, key, keylen, 1)) + if (ep11_check_aes_key_with_hdr(pkey_dbf_info, + 3, key, keylen, 1)) return -EINVAL; } else if (hdr->type == TOKTYPE_NON_CCA && hdr->version == TOKVER_EP11_ECC_WITH_HEADER && is_ep11_keyblob(key + sizeof(struct ep11kblob_header))) { /* EP11 ECC key blob with header */ - if (ep11_check_ecc_key_with_hdr(debug_info, 3, key, keylen, 1)) + if (ep11_check_ecc_key_with_hdr(pkey_dbf_info, + 3, key, keylen, 1)) return -EINVAL; } else if (hdr->type == TOKTYPE_NON_CCA && hdr->version == TOKVER_EP11_AES && is_ep11_keyblob(key)) { /* EP11 AES key blob with header in session field */ - if (ep11_check_aes_key(debug_info, 3, key, keylen, 1)) + if (ep11_check_aes_key(pkey_dbf_info, 3, key, keylen, 1)) return -EINVAL; } else if (hdr->type == TOKTYPE_CCA_INTERNAL) { if (hdr->version == TOKVER_CCA_AES) { /* CCA AES data key */ if (keylen != sizeof(struct secaeskeytoken)) return -EINVAL; - if (cca_check_secaeskeytoken(debug_info, 3, key, 0)) + if (cca_check_secaeskeytoken(pkey_dbf_info, 3, key, 0)) return -EINVAL; } else if (hdr->version == TOKVER_CCA_VLSC) { /* CCA AES cipher key */ if (keylen < hdr->len || keylen > MAXCCAVLSCTOKENSIZE) return -EINVAL; - if (cca_check_secaescipherkey(debug_info, 3, key, 0, 1)) + if (cca_check_secaescipherkey(pkey_dbf_info, + 3, key, 0, 1)) return -EINVAL; } else { - DEBUG_ERR("%s unknown CCA internal token version %d\n", - __func__, hdr->version); + PKEY_DBF_ERR("%s unknown CCA internal token version %d\n", + __func__, hdr->version); return -EINVAL; } } else if (hdr->type == TOKTYPE_CCA_INTERNAL_PKA) { /* CCA ECC (private) key */ if (keylen < sizeof(struct eccprivkeytoken)) return -EINVAL; - if (cca_check_sececckeytoken(debug_info, 3, key, keylen, 1)) + if (cca_check_sececckeytoken(pkey_dbf_info, 3, key, keylen, 1)) return -EINVAL; } else if (hdr->type == TOKTYPE_NON_CCA) { return pkey_nonccatok2pkey(key, keylen, protkey, protkeylen, protkeytype); } else { - DEBUG_ERR("%s unknown/unsupported blob type %d\n", - __func__, hdr->type); + PKEY_DBF_ERR("%s unknown/unsupported blob type %d\n", + __func__, hdr->type); return -EINVAL; } @@ -1350,7 +1358,7 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, return -EFAULT; rc = cca_genseckey(kgs.cardnr, kgs.domain, kgs.keytype, kgs.seckey.seckey); - DEBUG_DBG("%s cca_genseckey()=%d\n", __func__, rc); + pr_debug("%s cca_genseckey()=%d\n", __func__, rc); if (rc) break; if (copy_to_user(ugs, &kgs, sizeof(kgs))) @@ -1365,7 +1373,7 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, return -EFAULT; rc = cca_clr2seckey(kcs.cardnr, kcs.domain, kcs.keytype, kcs.clrkey.clrkey, kcs.seckey.seckey); - DEBUG_DBG("%s cca_clr2seckey()=%d\n", __func__, rc); + pr_debug("%s cca_clr2seckey()=%d\n", __func__, rc); if (rc) break; if (copy_to_user(ucs, &kcs, sizeof(kcs))) @@ -1383,7 +1391,7 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, rc = cca_sec2protkey(ksp.cardnr, ksp.domain, ksp.seckey.seckey, ksp.protkey.protkey, &ksp.protkey.len, &ksp.protkey.type); - DEBUG_DBG("%s cca_sec2protkey()=%d\n", __func__, rc); + pr_debug("%s cca_sec2protkey()=%d\n", __func__, rc); if (rc) break; if (copy_to_user(usp, &ksp, sizeof(ksp))) @@ -1400,7 +1408,7 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, rc = pkey_clr2protkey(kcp.keytype, kcp.clrkey.clrkey, kcp.protkey.protkey, &kcp.protkey.len, &kcp.protkey.type); - DEBUG_DBG("%s pkey_clr2protkey()=%d\n", __func__, rc); + pr_debug("%s pkey_clr2protkey()=%d\n", __func__, rc); if (rc) break; if (copy_to_user(ucp, &kcp, sizeof(kcp))) @@ -1416,7 +1424,7 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, return -EFAULT; rc = cca_findcard(kfc.seckey.seckey, &kfc.cardnr, &kfc.domain, 1); - DEBUG_DBG("%s cca_findcard()=%d\n", __func__, rc); + pr_debug("%s cca_findcard()=%d\n", __func__, rc); if (rc < 0) break; if (copy_to_user(ufc, &kfc, sizeof(kfc))) @@ -1432,7 +1440,7 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, ksp.protkey.len = sizeof(ksp.protkey.protkey); rc = pkey_skey2pkey(ksp.seckey.seckey, ksp.protkey.protkey, &ksp.protkey.len, &ksp.protkey.type); - DEBUG_DBG("%s pkey_skey2pkey()=%d\n", __func__, rc); + pr_debug("%s pkey_skey2pkey()=%d\n", __func__, rc); if (rc) break; if (copy_to_user(usp, &ksp, sizeof(ksp))) @@ -1447,7 +1455,7 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, return -EFAULT; rc = pkey_verifykey(&kvk.seckey, &kvk.cardnr, &kvk.domain, &kvk.keysize, &kvk.attributes); - DEBUG_DBG("%s pkey_verifykey()=%d\n", __func__, rc); + pr_debug("%s pkey_verifykey()=%d\n", __func__, rc); if (rc) break; if (copy_to_user(uvk, &kvk, sizeof(kvk))) @@ -1463,7 +1471,7 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, kgp.protkey.len = sizeof(kgp.protkey.protkey); rc = pkey_genprotkey(kgp.keytype, kgp.protkey.protkey, &kgp.protkey.len, &kgp.protkey.type); - DEBUG_DBG("%s pkey_genprotkey()=%d\n", __func__, rc); + pr_debug("%s pkey_genprotkey()=%d\n", __func__, rc); if (rc) break; if (copy_to_user(ugp, &kgp, sizeof(kgp))) @@ -1478,7 +1486,7 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, return -EFAULT; rc = pkey_verifyprotkey(kvp.protkey.protkey, kvp.protkey.len, kvp.protkey.type); - DEBUG_DBG("%s pkey_verifyprotkey()=%d\n", __func__, rc); + pr_debug("%s pkey_verifyprotkey()=%d\n", __func__, rc); break; } case PKEY_KBLOB2PROTK: { @@ -1494,7 +1502,7 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, ktp.protkey.len = sizeof(ktp.protkey.protkey); rc = pkey_keyblob2pkey(kkey, ktp.keylen, ktp.protkey.protkey, &ktp.protkey.len, &ktp.protkey.type); - DEBUG_DBG("%s pkey_keyblob2pkey()=%d\n", __func__, rc); + pr_debug("%s pkey_keyblob2pkey()=%d\n", __func__, rc); memzero_explicit(kkey, ktp.keylen); kfree(kkey); if (rc) @@ -1523,7 +1531,7 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, rc = pkey_genseckey2(apqns, kgs.apqn_entries, kgs.type, kgs.size, kgs.keygenflags, kkey, &klen); - DEBUG_DBG("%s pkey_genseckey2()=%d\n", __func__, rc); + pr_debug("%s pkey_genseckey2()=%d\n", __func__, rc); kfree(apqns); if (rc) { kfree(kkey); @@ -1565,7 +1573,7 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, rc = pkey_clr2seckey2(apqns, kcs.apqn_entries, kcs.type, kcs.size, kcs.keygenflags, kcs.clrkey.clrkey, kkey, &klen); - DEBUG_DBG("%s pkey_clr2seckey2()=%d\n", __func__, rc); + pr_debug("%s pkey_clr2seckey2()=%d\n", __func__, rc); kfree(apqns); if (rc) { kfree(kkey); @@ -1601,7 +1609,7 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, rc = pkey_verifykey2(kkey, kvk.keylen, &kvk.cardnr, &kvk.domain, &kvk.type, &kvk.size, &kvk.flags); - DEBUG_DBG("%s pkey_verifykey2()=%d\n", __func__, rc); + pr_debug("%s pkey_verifykey2()=%d\n", __func__, rc); kfree(kkey); if (rc) break; @@ -1630,7 +1638,7 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, kkey, ktp.keylen, ktp.protkey.protkey, &ktp.protkey.len, &ktp.protkey.type); - DEBUG_DBG("%s pkey_keyblob2pkey2()=%d\n", __func__, rc); + pr_debug("%s pkey_keyblob2pkey2()=%d\n", __func__, rc); kfree(apqns); memzero_explicit(kkey, ktp.keylen); kfree(kkey); @@ -1664,7 +1672,7 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, } rc = pkey_apqns4key(kkey, kak.keylen, kak.flags, apqns, &nr_apqns); - DEBUG_DBG("%s pkey_apqns4key()=%d\n", __func__, rc); + pr_debug("%s pkey_apqns4key()=%d\n", __func__, rc); kfree(kkey); if (rc && rc != -ENOSPC) { kfree(apqns); @@ -1707,7 +1715,7 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, } rc = pkey_apqns4keytype(kat.type, kat.cur_mkvp, kat.alt_mkvp, kat.flags, apqns, &nr_apqns); - DEBUG_DBG("%s pkey_apqns4keytype()=%d\n", __func__, rc); + pr_debug("%s pkey_apqns4keytype()=%d\n", __func__, rc); if (rc && rc != -ENOSPC) { kfree(apqns); break; @@ -1757,7 +1765,7 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, rc = pkey_keyblob2pkey3(apqns, ktp.apqn_entries, kkey, ktp.keylen, protkey, &protkeylen, &ktp.pkeytype); - DEBUG_DBG("%s pkey_keyblob2pkey3()=%d\n", __func__, rc); + pr_debug("%s pkey_keyblob2pkey3()=%d\n", __func__, rc); kfree(apqns); memzero_explicit(kkey, ktp.keylen); kfree(kkey); diff --git a/drivers/s390/crypto/vfio_ap_drv.c b/drivers/s390/crypto/vfio_ap_drv.c index a5ab03e42ff1..4aeb3e1213c7 100644 --- a/drivers/s390/crypto/vfio_ap_drv.c +++ b/drivers/s390/crypto/vfio_ap_drv.c @@ -60,7 +60,7 @@ static void vfio_ap_matrix_dev_release(struct device *dev) kfree(matrix_dev); } -static struct bus_type matrix_bus = { +static const struct bus_type matrix_bus = { .name = "matrix", }; diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c index 983b3b16196c..fc169bc61593 100644 --- a/drivers/s390/crypto/vfio_ap_ops.c +++ b/drivers/s390/crypto/vfio_ap_ops.c @@ -659,6 +659,21 @@ static bool vfio_ap_mdev_filter_cdoms(struct ap_matrix_mdev *matrix_mdev) AP_DOMAINS); } +static bool _queue_passable(struct vfio_ap_queue *q) +{ + if (!q) + return false; + + switch (q->reset_status.response_code) { + case AP_RESPONSE_NORMAL: + case AP_RESPONSE_DECONFIGURED: + case AP_RESPONSE_CHECKSTOPPED: + return true; + default: + return false; + } +} + /* * vfio_ap_mdev_filter_matrix - filter the APQNs assigned to the matrix mdev * to ensure no queue devices are passed through to @@ -687,7 +702,6 @@ static bool vfio_ap_mdev_filter_matrix(struct ap_matrix_mdev *matrix_mdev, unsigned long apid, apqi, apqn; DECLARE_BITMAP(prev_shadow_apm, AP_DEVICES); DECLARE_BITMAP(prev_shadow_aqm, AP_DOMAINS); - struct vfio_ap_queue *q; bitmap_copy(prev_shadow_apm, matrix_mdev->shadow_apcb.apm, AP_DEVICES); bitmap_copy(prev_shadow_aqm, matrix_mdev->shadow_apcb.aqm, AP_DOMAINS); @@ -716,8 +730,7 @@ static bool vfio_ap_mdev_filter_matrix(struct ap_matrix_mdev *matrix_mdev, * hardware device. */ apqn = AP_MKQID(apid, apqi); - q = vfio_ap_mdev_get_queue(matrix_mdev, apqn); - if (!q || q->reset_status.response_code) { + if (!_queue_passable(vfio_ap_mdev_get_queue(matrix_mdev, apqn))) { clear_bit_inv(apid, matrix_mdev->shadow_apcb.apm); /* @@ -1691,6 +1704,7 @@ static int apq_status_check(int apqn, struct ap_queue_status *status) switch (status->response_code) { case AP_RESPONSE_NORMAL: case AP_RESPONSE_DECONFIGURED: + case AP_RESPONSE_CHECKSTOPPED: return 0; case AP_RESPONSE_RESET_IN_PROGRESS: case AP_RESPONSE_BUSY: @@ -1747,14 +1761,6 @@ static void apq_reset_check(struct work_struct *reset_work) memcpy(&q->reset_status, &status, sizeof(status)); continue; } - /* - * When an AP adapter is deconfigured, the - * associated queues are reset, so let's set the - * status response code to 0 so the queue may be - * passed through (i.e., not filtered) - */ - if (status.response_code == AP_RESPONSE_DECONFIGURED) - q->reset_status.response_code = 0; if (q->saved_isc != VFIO_AP_ISC_INVALID) vfio_ap_free_aqic_resources(q); break; @@ -1781,12 +1787,7 @@ static void vfio_ap_mdev_reset_queue(struct vfio_ap_queue *q) queue_work(system_long_wq, &q->reset_work); break; case AP_RESPONSE_DECONFIGURED: - /* - * When an AP adapter is deconfigured, the associated - * queues are reset, so let's set the status response code to 0 - * so the queue may be passed through (i.e., not filtered). - */ - q->reset_status.response_code = 0; + case AP_RESPONSE_CHECKSTOPPED: vfio_ap_free_aqic_resources(q); break; default: diff --git a/drivers/s390/crypto/zcrypt_api.c b/drivers/s390/crypto/zcrypt_api.c index 74200f54dfff..02c503f16bc2 100644 --- a/drivers/s390/crypto/zcrypt_api.c +++ b/drivers/s390/crypto/zcrypt_api.c @@ -12,6 +12,9 @@ * Multiple device nodes: Harald Freudenberger <freude@linux.ibm.com> */ +#define KMSG_COMPONENT "zcrypt" +#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt + #include <linux/module.h> #include <linux/init.h> #include <linux/interrupt.h> @@ -57,10 +60,6 @@ DEFINE_SPINLOCK(zcrypt_list_lock); LIST_HEAD(zcrypt_card_list); static atomic_t zcrypt_open_count = ATOMIC_INIT(0); -static atomic_t zcrypt_rescan_count = ATOMIC_INIT(0); - -atomic_t zcrypt_rescan_req = ATOMIC_INIT(0); -EXPORT_SYMBOL(zcrypt_rescan_req); static LIST_HEAD(zcrypt_ops_list); @@ -69,20 +68,15 @@ debug_info_t *zcrypt_dbf_info; /* * Process a rescan of the transport layer. - * - * Returns 1, if the rescan has been processed, otherwise 0. + * Runs a synchronous AP bus rescan. + * Returns true if something has changed (for example the + * bus scan has found and build up new devices) and it is + * worth to do a retry. Otherwise false is returned meaning + * no changes on the AP bus level. */ -static inline int zcrypt_process_rescan(void) -{ - if (atomic_read(&zcrypt_rescan_req)) { - atomic_set(&zcrypt_rescan_req, 0); - atomic_inc(&zcrypt_rescan_count); - ap_bus_force_rescan(); - ZCRYPT_DBF_INFO("%s rescan count=%07d\n", __func__, - atomic_inc_return(&zcrypt_rescan_count)); - return 1; - } - return 0; +static inline bool zcrypt_process_rescan(void) +{ + return ap_bus_force_rescan(); } void zcrypt_msgtype_register(struct zcrypt_ops *zops) @@ -715,8 +709,7 @@ static long zcrypt_rsa_modexpo(struct ap_perms *perms, spin_unlock(&zcrypt_list_lock); if (!pref_zq) { - ZCRYPT_DBF_DBG("%s no matching queue found => ENODEV\n", - __func__); + pr_debug("%s no matching queue found => ENODEV\n", __func__); rc = -ENODEV; goto out; } @@ -820,8 +813,7 @@ static long zcrypt_rsa_crt(struct ap_perms *perms, spin_unlock(&zcrypt_list_lock); if (!pref_zq) { - ZCRYPT_DBF_DBG("%s no matching queue found => ENODEV\n", - __func__); + pr_debug("%s no matching queue found => ENODEV\n", __func__); rc = -ENODEV; goto out; } @@ -865,6 +857,8 @@ static long _zcrypt_send_cprb(bool userspace, struct ap_perms *perms, rc = prep_cca_ap_msg(userspace, xcrb, &ap_msg, &func_code, &domain); if (rc) goto out; + print_hex_dump_debug("ccareq: ", DUMP_PREFIX_ADDRESS, 16, 1, + ap_msg.msg, ap_msg.len, false); tdom = *domain; if (perms != &ap_perms && tdom < AP_DOMAINS) { @@ -940,8 +934,8 @@ static long _zcrypt_send_cprb(bool userspace, struct ap_perms *perms, spin_unlock(&zcrypt_list_lock); if (!pref_zq) { - ZCRYPT_DBF_DBG("%s no match for address %02x.%04x => ENODEV\n", - __func__, xcrb->user_defined, *domain); + pr_debug("%s no match for address %02x.%04x => ENODEV\n", + __func__, xcrb->user_defined, *domain); rc = -ENODEV; goto out; } @@ -952,6 +946,10 @@ static long _zcrypt_send_cprb(bool userspace, struct ap_perms *perms, *domain = AP_QID_QUEUE(qid); rc = pref_zq->ops->send_cprb(userspace, pref_zq, xcrb, &ap_msg); + if (!rc) { + print_hex_dump_debug("ccarpl: ", DUMP_PREFIX_ADDRESS, 16, 1, + ap_msg.msg, ap_msg.len, false); + } spin_lock(&zcrypt_list_lock); zcrypt_drop_queue(pref_zc, pref_zq, mod, wgt); @@ -970,7 +968,26 @@ out: long zcrypt_send_cprb(struct ica_xcRB *xcrb) { - return _zcrypt_send_cprb(false, &ap_perms, NULL, xcrb); + struct zcrypt_track tr; + int rc; + + memset(&tr, 0, sizeof(tr)); + + do { + rc = _zcrypt_send_cprb(false, &ap_perms, &tr, xcrb); + } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); + + /* on ENODEV failure: retry once again after a requested rescan */ + if (rc == -ENODEV && zcrypt_process_rescan()) + do { + rc = _zcrypt_send_cprb(false, &ap_perms, &tr, xcrb); + } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); + if (rc == -EAGAIN && tr.again_counter >= TRACK_AGAIN_MAX) + rc = -EIO; + if (rc) + pr_debug("%s rc=%d\n", __func__, rc); + + return rc; } EXPORT_SYMBOL(zcrypt_send_cprb); @@ -1045,6 +1062,8 @@ static long _zcrypt_send_ep11_cprb(bool userspace, struct ap_perms *perms, rc = prep_ep11_ap_msg(userspace, xcrb, &ap_msg, &func_code, &domain); if (rc) goto out_free; + print_hex_dump_debug("ep11req: ", DUMP_PREFIX_ADDRESS, 16, 1, + ap_msg.msg, ap_msg.len, false); if (perms != &ap_perms && domain < AUTOSEL_DOM) { if (ap_msg.flags & AP_MSG_FLAG_ADMIN) { @@ -1113,15 +1132,15 @@ static long _zcrypt_send_ep11_cprb(bool userspace, struct ap_perms *perms, if (!pref_zq) { if (targets && target_num == 1) { - ZCRYPT_DBF_DBG("%s no match for address %02x.%04x => ENODEV\n", - __func__, (int)targets->ap_id, - (int)targets->dom_id); + pr_debug("%s no match for address %02x.%04x => ENODEV\n", + __func__, (int)targets->ap_id, + (int)targets->dom_id); } else if (targets) { - ZCRYPT_DBF_DBG("%s no match for %d target addrs => ENODEV\n", - __func__, (int)target_num); + pr_debug("%s no match for %d target addrs => ENODEV\n", + __func__, (int)target_num); } else { - ZCRYPT_DBF_DBG("%s no match for address ff.ffff => ENODEV\n", - __func__); + pr_debug("%s no match for address ff.ffff => ENODEV\n", + __func__); } rc = -ENODEV; goto out_free; @@ -1129,6 +1148,10 @@ static long _zcrypt_send_ep11_cprb(bool userspace, struct ap_perms *perms, qid = pref_zq->queue->qid; rc = pref_zq->ops->send_ep11_cprb(userspace, pref_zq, xcrb, &ap_msg); + if (!rc) { + print_hex_dump_debug("ep11rpl: ", DUMP_PREFIX_ADDRESS, 16, 1, + ap_msg.msg, ap_msg.len, false); + } spin_lock(&zcrypt_list_lock); zcrypt_drop_queue(pref_zc, pref_zq, mod, wgt); @@ -1149,7 +1172,26 @@ out: long zcrypt_send_ep11_cprb(struct ep11_urb *xcrb) { - return _zcrypt_send_ep11_cprb(false, &ap_perms, NULL, xcrb); + struct zcrypt_track tr; + int rc; + + memset(&tr, 0, sizeof(tr)); + + do { + rc = _zcrypt_send_ep11_cprb(false, &ap_perms, &tr, xcrb); + } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); + + /* on ENODEV failure: retry once again after a requested rescan */ + if (rc == -ENODEV && zcrypt_process_rescan()) + do { + rc = _zcrypt_send_ep11_cprb(false, &ap_perms, &tr, xcrb); + } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); + if (rc == -EAGAIN && tr.again_counter >= TRACK_AGAIN_MAX) + rc = -EIO; + if (rc) + pr_debug("%s rc=%d\n", __func__, rc); + + return rc; } EXPORT_SYMBOL(zcrypt_send_ep11_cprb); @@ -1199,8 +1241,7 @@ static long zcrypt_rng(char *buffer) spin_unlock(&zcrypt_list_lock); if (!pref_zq) { - ZCRYPT_DBF_DBG("%s no matching queue found => ENODEV\n", - __func__); + pr_debug("%s no matching queue found => ENODEV\n", __func__); rc = -ENODEV; goto out; } @@ -1431,20 +1472,17 @@ static int icarsamodexpo_ioctl(struct ap_perms *perms, unsigned long arg) do { rc = zcrypt_rsa_modexpo(perms, &tr, &mex); - if (rc == -EAGAIN) - tr.again_counter++; - } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); - /* on failure: retry once again after a requested rescan */ - if ((rc == -ENODEV) && (zcrypt_process_rescan())) + } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); + + /* on ENODEV failure: retry once again after a requested rescan */ + if (rc == -ENODEV && zcrypt_process_rescan()) do { rc = zcrypt_rsa_modexpo(perms, &tr, &mex); - if (rc == -EAGAIN) - tr.again_counter++; - } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); + } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); if (rc == -EAGAIN && tr.again_counter >= TRACK_AGAIN_MAX) rc = -EIO; if (rc) { - ZCRYPT_DBF_DBG("ioctl ICARSAMODEXPO rc=%d\n", rc); + pr_debug("ioctl ICARSAMODEXPO rc=%d\n", rc); return rc; } return put_user(mex.outputdatalength, &umex->outputdatalength); @@ -1463,20 +1501,17 @@ static int icarsacrt_ioctl(struct ap_perms *perms, unsigned long arg) do { rc = zcrypt_rsa_crt(perms, &tr, &crt); - if (rc == -EAGAIN) - tr.again_counter++; - } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); - /* on failure: retry once again after a requested rescan */ - if ((rc == -ENODEV) && (zcrypt_process_rescan())) + } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); + + /* on ENODEV failure: retry once again after a requested rescan */ + if (rc == -ENODEV && zcrypt_process_rescan()) do { rc = zcrypt_rsa_crt(perms, &tr, &crt); - if (rc == -EAGAIN) - tr.again_counter++; - } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); + } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); if (rc == -EAGAIN && tr.again_counter >= TRACK_AGAIN_MAX) rc = -EIO; if (rc) { - ZCRYPT_DBF_DBG("ioctl ICARSACRT rc=%d\n", rc); + pr_debug("ioctl ICARSACRT rc=%d\n", rc); return rc; } return put_user(crt.outputdatalength, &ucrt->outputdatalength); @@ -1495,21 +1530,18 @@ static int zsecsendcprb_ioctl(struct ap_perms *perms, unsigned long arg) do { rc = _zcrypt_send_cprb(true, perms, &tr, &xcrb); - if (rc == -EAGAIN) - tr.again_counter++; - } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); - /* on failure: retry once again after a requested rescan */ - if ((rc == -ENODEV) && (zcrypt_process_rescan())) + } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); + + /* on ENODEV failure: retry once again after a requested rescan */ + if (rc == -ENODEV && zcrypt_process_rescan()) do { rc = _zcrypt_send_cprb(true, perms, &tr, &xcrb); - if (rc == -EAGAIN) - tr.again_counter++; - } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); + } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); if (rc == -EAGAIN && tr.again_counter >= TRACK_AGAIN_MAX) rc = -EIO; if (rc) - ZCRYPT_DBF_DBG("ioctl ZSENDCPRB rc=%d status=0x%x\n", - rc, xcrb.status); + pr_debug("ioctl ZSENDCPRB rc=%d status=0x%x\n", + rc, xcrb.status); if (copy_to_user(uxcrb, &xcrb, sizeof(xcrb))) return -EFAULT; return rc; @@ -1528,20 +1560,17 @@ static int zsendep11cprb_ioctl(struct ap_perms *perms, unsigned long arg) do { rc = _zcrypt_send_ep11_cprb(true, perms, &tr, &xcrb); - if (rc == -EAGAIN) - tr.again_counter++; - } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); - /* on failure: retry once again after a requested rescan */ - if ((rc == -ENODEV) && (zcrypt_process_rescan())) + } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); + + /* on ENODEV failure: retry once again after a requested rescan */ + if (rc == -ENODEV && zcrypt_process_rescan()) do { rc = _zcrypt_send_ep11_cprb(true, perms, &tr, &xcrb); - if (rc == -EAGAIN) - tr.again_counter++; - } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); + } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); if (rc == -EAGAIN && tr.again_counter >= TRACK_AGAIN_MAX) rc = -EIO; if (rc) - ZCRYPT_DBF_DBG("ioctl ZSENDEP11CPRB rc=%d\n", rc); + pr_debug("ioctl ZSENDEP11CPRB rc=%d\n", rc); if (copy_to_user(uxcrb, &xcrb, sizeof(xcrb))) return -EFAULT; return rc; @@ -1670,7 +1699,7 @@ static long zcrypt_unlocked_ioctl(struct file *filp, unsigned int cmd, } /* unknown ioctl number */ default: - ZCRYPT_DBF_DBG("unknown ioctl 0x%08x\n", cmd); + pr_debug("unknown ioctl 0x%08x\n", cmd); return -ENOIOCTLCMD; } } @@ -1708,16 +1737,13 @@ static long trans_modexpo32(struct ap_perms *perms, struct file *filp, mex64.n_modulus = compat_ptr(mex32.n_modulus); do { rc = zcrypt_rsa_modexpo(perms, &tr, &mex64); - if (rc == -EAGAIN) - tr.again_counter++; - } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); - /* on failure: retry once again after a requested rescan */ - if ((rc == -ENODEV) && (zcrypt_process_rescan())) + } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); + + /* on ENODEV failure: retry once again after a requested rescan */ + if (rc == -ENODEV && zcrypt_process_rescan()) do { rc = zcrypt_rsa_modexpo(perms, &tr, &mex64); - if (rc == -EAGAIN) - tr.again_counter++; - } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); + } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); if (rc == -EAGAIN && tr.again_counter >= TRACK_AGAIN_MAX) rc = -EIO; if (rc) @@ -1761,16 +1787,13 @@ static long trans_modexpo_crt32(struct ap_perms *perms, struct file *filp, crt64.u_mult_inv = compat_ptr(crt32.u_mult_inv); do { rc = zcrypt_rsa_crt(perms, &tr, &crt64); - if (rc == -EAGAIN) - tr.again_counter++; - } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); - /* on failure: retry once again after a requested rescan */ - if ((rc == -ENODEV) && (zcrypt_process_rescan())) + } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); + + /* on ENODEV failure: retry once again after a requested rescan */ + if (rc == -ENODEV && zcrypt_process_rescan()) do { rc = zcrypt_rsa_crt(perms, &tr, &crt64); - if (rc == -EAGAIN) - tr.again_counter++; - } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); + } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); if (rc == -EAGAIN && tr.again_counter >= TRACK_AGAIN_MAX) rc = -EIO; if (rc) @@ -1833,16 +1856,13 @@ static long trans_xcrb32(struct ap_perms *perms, struct file *filp, xcrb64.status = xcrb32.status; do { rc = _zcrypt_send_cprb(true, perms, &tr, &xcrb64); - if (rc == -EAGAIN) - tr.again_counter++; - } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); - /* on failure: retry once again after a requested rescan */ - if ((rc == -ENODEV) && (zcrypt_process_rescan())) + } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); + + /* on ENODEV failure: retry once again after a requested rescan */ + if (rc == -ENODEV && zcrypt_process_rescan()) do { rc = _zcrypt_send_cprb(true, perms, &tr, &xcrb64); - if (rc == -EAGAIN) - tr.again_counter++; - } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); + } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); if (rc == -EAGAIN && tr.again_counter >= TRACK_AGAIN_MAX) rc = -EIO; xcrb32.reply_control_blk_length = xcrb64.reply_control_blk_length; @@ -1914,8 +1934,8 @@ static int zcrypt_rng_data_read(struct hwrng *rng, u32 *data) */ if (zcrypt_rng_buffer_index == 0) { rc = zcrypt_rng((char *)zcrypt_rng_buffer); - /* on failure: retry once again after a requested rescan */ - if ((rc == -ENODEV) && (zcrypt_process_rescan())) + /* on ENODEV failure: retry once again after an AP bus rescan */ + if (rc == -ENODEV && zcrypt_process_rescan()) rc = zcrypt_rng((char *)zcrypt_rng_buffer); if (rc < 0) return -EIO; @@ -1977,7 +1997,7 @@ void zcrypt_rng_device_remove(void) * an asynchronous job. This function waits until these initial jobs * are done and so the zcrypt api should be ready to serve crypto * requests - if there are resources available. The function uses an - * internal timeout of 60s. The very first caller will either wait for + * internal timeout of 30s. The very first caller will either wait for * ap bus bindings complete or the timeout happens. This state will be * remembered for further callers which will only be blocked until a * decision is made (timeout or bindings complete). @@ -1996,8 +2016,8 @@ int zcrypt_wait_api_operational(void) switch (zcrypt_wait_api_state) { case 0: /* initial state, invoke wait for the ap bus complete */ - rc = ap_wait_init_apqn_bindings_complete( - msecs_to_jiffies(60 * 1000)); + rc = ap_wait_apqn_bindings_complete( + msecs_to_jiffies(ZCRYPT_WAIT_BINDINGS_COMPLETE_MS)); switch (rc) { case 0: /* ap bus bindings are complete */ @@ -2014,8 +2034,8 @@ int zcrypt_wait_api_operational(void) break; default: /* other failure */ - ZCRYPT_DBF_DBG("%s ap_wait_init_apqn_bindings_complete()=%d\n", - __func__, rc); + pr_debug("%s ap_wait_init_apqn_bindings_complete()=%d\n", + __func__, rc); break; } break; @@ -2038,7 +2058,7 @@ EXPORT_SYMBOL(zcrypt_wait_api_operational); int __init zcrypt_debug_init(void) { zcrypt_dbf_info = debug_register("zcrypt", 2, 1, - DBF_MAX_SPRINTF_ARGS * sizeof(long)); + ZCRYPT_DBF_MAX_SPRINTF_ARGS * sizeof(long)); debug_register_view(zcrypt_dbf_info, &debug_sprintf_view); debug_set_level(zcrypt_dbf_info, DBF_ERR); diff --git a/drivers/s390/crypto/zcrypt_api.h b/drivers/s390/crypto/zcrypt_api.h index de659954c8f7..4ed481df57ca 100644 --- a/drivers/s390/crypto/zcrypt_api.h +++ b/drivers/s390/crypto/zcrypt_api.h @@ -38,6 +38,15 @@ */ #define ZCRYPT_RNG_BUFFER_SIZE 4096 +/** + * The zcrypt_wait_api_operational() function waits this + * amount in milliseconds for ap_wait_aqpn_bindings_complete(). + * Also on a cprb send failure with ENODEV the send functions + * trigger an ap bus rescan and wait this time in milliseconds + * for ap_wait_aqpn_bindings_complete() before resending. + */ +#define ZCRYPT_WAIT_BINDINGS_COMPLETE_MS 30000 + /* * Identifier for Crypto Request Performance Index */ diff --git a/drivers/s390/crypto/zcrypt_ccamisc.c b/drivers/s390/crypto/zcrypt_ccamisc.c index 263fe182648b..0a3a678ffc7e 100644 --- a/drivers/s390/crypto/zcrypt_ccamisc.c +++ b/drivers/s390/crypto/zcrypt_ccamisc.c @@ -23,11 +23,6 @@ #include "zcrypt_msgtype6.h" #include "zcrypt_ccamisc.h" -#define DEBUG_DBG(...) ZCRYPT_DBF(DBF_DEBUG, ##__VA_ARGS__) -#define DEBUG_INFO(...) ZCRYPT_DBF(DBF_INFO, ##__VA_ARGS__) -#define DEBUG_WARN(...) ZCRYPT_DBF(DBF_WARN, ##__VA_ARGS__) -#define DEBUG_ERR(...) ZCRYPT_DBF(DBF_ERR, ##__VA_ARGS__) - /* Size of parameter block used for all cca requests/replies */ #define PARMBSIZE 512 @@ -367,8 +362,8 @@ int cca_genseckey(u16 cardnr, u16 domain, memcpy(preqparm->lv1.key_length, "KEYLN32 ", 8); break; default: - DEBUG_ERR("%s unknown/unsupported keybitsize %d\n", - __func__, keybitsize); + ZCRYPT_DBF_ERR("%s unknown/unsupported keybitsize %d\n", + __func__, keybitsize); rc = -EINVAL; goto out; } @@ -386,15 +381,15 @@ int cca_genseckey(u16 cardnr, u16 domain, /* forward xcrb with request CPRB and reply CPRB to zcrypt dd */ rc = zcrypt_send_cprb(&xcrb); if (rc) { - DEBUG_ERR("%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, errno %d\n", - __func__, (int)cardnr, (int)domain, rc); + ZCRYPT_DBF_ERR("%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, errno %d\n", + __func__, (int)cardnr, (int)domain, rc); goto out; } /* check response returncode and reasoncode */ if (prepcblk->ccp_rtcode != 0) { - DEBUG_ERR("%s secure key generate failure, card response %d/%d\n", - __func__, + ZCRYPT_DBF_ERR("%s secure key generate failure, card response %d/%d\n", + __func__, (int)prepcblk->ccp_rtcode, (int)prepcblk->ccp_rscode); rc = -EIO; @@ -411,8 +406,8 @@ int cca_genseckey(u16 cardnr, u16 domain, - sizeof(prepparm->lv3.keyblock.toklen) - sizeof(prepparm->lv3.keyblock.tokattr); if (seckeysize != SECKEYBLOBSIZE) { - DEBUG_ERR("%s secure token size mismatch %d != %d bytes\n", - __func__, seckeysize, SECKEYBLOBSIZE); + ZCRYPT_DBF_ERR("%s secure token size mismatch %d != %d bytes\n", + __func__, seckeysize, SECKEYBLOBSIZE); rc = -EIO; goto out; } @@ -505,8 +500,8 @@ int cca_clr2seckey(u16 cardnr, u16 domain, u32 keybitsize, keysize = 32; break; default: - DEBUG_ERR("%s unknown/unsupported keybitsize %d\n", - __func__, keybitsize); + ZCRYPT_DBF_ERR("%s unknown/unsupported keybitsize %d\n", + __func__, keybitsize); rc = -EINVAL; goto out; } @@ -524,17 +519,17 @@ int cca_clr2seckey(u16 cardnr, u16 domain, u32 keybitsize, /* forward xcrb with request CPRB and reply CPRB to zcrypt dd */ rc = zcrypt_send_cprb(&xcrb); if (rc) { - DEBUG_ERR("%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n", - __func__, (int)cardnr, (int)domain, rc); + ZCRYPT_DBF_ERR("%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n", + __func__, (int)cardnr, (int)domain, rc); goto out; } /* check response returncode and reasoncode */ if (prepcblk->ccp_rtcode != 0) { - DEBUG_ERR("%s clear key import failure, card response %d/%d\n", - __func__, - (int)prepcblk->ccp_rtcode, - (int)prepcblk->ccp_rscode); + ZCRYPT_DBF_ERR("%s clear key import failure, card response %d/%d\n", + __func__, + (int)prepcblk->ccp_rtcode, + (int)prepcblk->ccp_rscode); rc = -EIO; goto out; } @@ -549,8 +544,8 @@ int cca_clr2seckey(u16 cardnr, u16 domain, u32 keybitsize, - sizeof(prepparm->lv3.keyblock.toklen) - sizeof(prepparm->lv3.keyblock.tokattr); if (seckeysize != SECKEYBLOBSIZE) { - DEBUG_ERR("%s secure token size mismatch %d != %d bytes\n", - __func__, seckeysize, SECKEYBLOBSIZE); + ZCRYPT_DBF_ERR("%s secure token size mismatch %d != %d bytes\n", + __func__, seckeysize, SECKEYBLOBSIZE); rc = -EIO; goto out; } @@ -651,17 +646,17 @@ int cca_sec2protkey(u16 cardnr, u16 domain, /* forward xcrb with request CPRB and reply CPRB to zcrypt dd */ rc = zcrypt_send_cprb(&xcrb); if (rc) { - DEBUG_ERR("%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n", - __func__, (int)cardnr, (int)domain, rc); + ZCRYPT_DBF_ERR("%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n", + __func__, (int)cardnr, (int)domain, rc); goto out; } /* check response returncode and reasoncode */ if (prepcblk->ccp_rtcode != 0) { - DEBUG_ERR("%s unwrap secure key failure, card response %d/%d\n", - __func__, - (int)prepcblk->ccp_rtcode, - (int)prepcblk->ccp_rscode); + ZCRYPT_DBF_ERR("%s unwrap secure key failure, card response %d/%d\n", + __func__, + (int)prepcblk->ccp_rtcode, + (int)prepcblk->ccp_rscode); if (prepcblk->ccp_rtcode == 8 && prepcblk->ccp_rscode == 2290) rc = -EAGAIN; else @@ -669,10 +664,10 @@ int cca_sec2protkey(u16 cardnr, u16 domain, goto out; } if (prepcblk->ccp_rscode != 0) { - DEBUG_WARN("%s unwrap secure key warning, card response %d/%d\n", - __func__, - (int)prepcblk->ccp_rtcode, - (int)prepcblk->ccp_rscode); + ZCRYPT_DBF_WARN("%s unwrap secure key warning, card response %d/%d\n", + __func__, + (int)prepcblk->ccp_rtcode, + (int)prepcblk->ccp_rscode); } /* process response cprb param block */ @@ -683,8 +678,8 @@ int cca_sec2protkey(u16 cardnr, u16 domain, /* check the returned keyblock */ if (prepparm->lv3.ckb.version != 0x01 && prepparm->lv3.ckb.version != 0x02) { - DEBUG_ERR("%s reply param keyblock version mismatch 0x%02x\n", - __func__, (int)prepparm->lv3.ckb.version); + ZCRYPT_DBF_ERR("%s reply param keyblock version mismatch 0x%02x\n", + __func__, (int)prepparm->lv3.ckb.version); rc = -EIO; goto out; } @@ -707,8 +702,8 @@ int cca_sec2protkey(u16 cardnr, u16 domain, *protkeytype = PKEY_KEYTYPE_AES_256; break; default: - DEBUG_ERR("%s unknown/unsupported keylen %d\n", - __func__, prepparm->lv3.ckb.len); + ZCRYPT_DBF_ERR("%s unknown/unsupported keylen %d\n", + __func__, prepparm->lv3.ckb.len); rc = -EIO; goto out; } @@ -840,9 +835,8 @@ int cca_gencipherkey(u16 cardnr, u16 domain, u32 keybitsize, u32 keygenflags, case 256: break; default: - DEBUG_ERR( - "%s unknown/unsupported keybitsize %d\n", - __func__, keybitsize); + ZCRYPT_DBF_ERR("%s unknown/unsupported keybitsize %d\n", + __func__, keybitsize); rc = -EINVAL; goto out; } @@ -880,19 +874,17 @@ int cca_gencipherkey(u16 cardnr, u16 domain, u32 keybitsize, u32 keygenflags, /* forward xcrb with request CPRB and reply CPRB to zcrypt dd */ rc = zcrypt_send_cprb(&xcrb); if (rc) { - DEBUG_ERR( - "%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n", - __func__, (int)cardnr, (int)domain, rc); + ZCRYPT_DBF_ERR("%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n", + __func__, (int)cardnr, (int)domain, rc); goto out; } /* check response returncode and reasoncode */ if (prepcblk->ccp_rtcode != 0) { - DEBUG_ERR( - "%s cipher key generate failure, card response %d/%d\n", - __func__, - (int)prepcblk->ccp_rtcode, - (int)prepcblk->ccp_rscode); + ZCRYPT_DBF_ERR("%s cipher key generate failure, card response %d/%d\n", + __func__, + (int)prepcblk->ccp_rtcode, + (int)prepcblk->ccp_rscode); rc = -EIO; goto out; } @@ -905,8 +897,8 @@ int cca_gencipherkey(u16 cardnr, u16 domain, u32 keybitsize, u32 keygenflags, /* do some plausibility checks on the key block */ if (prepparm->kb.len < 120 + 5 * sizeof(uint16_t) || prepparm->kb.len > 136 + 5 * sizeof(uint16_t)) { - DEBUG_ERR("%s reply with invalid or unknown key block\n", - __func__); + ZCRYPT_DBF_ERR("%s reply with invalid or unknown key block\n", + __func__); rc = -EIO; goto out; } @@ -1048,19 +1040,17 @@ static int _ip_cprb_helper(u16 cardnr, u16 domain, /* forward xcrb with request CPRB and reply CPRB to zcrypt dd */ rc = zcrypt_send_cprb(&xcrb); if (rc) { - DEBUG_ERR( - "%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n", - __func__, (int)cardnr, (int)domain, rc); + ZCRYPT_DBF_ERR("%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n", + __func__, (int)cardnr, (int)domain, rc); goto out; } /* check response returncode and reasoncode */ if (prepcblk->ccp_rtcode != 0) { - DEBUG_ERR( - "%s CSNBKPI2 failure, card response %d/%d\n", - __func__, - (int)prepcblk->ccp_rtcode, - (int)prepcblk->ccp_rscode); + ZCRYPT_DBF_ERR("%s CSNBKPI2 failure, card response %d/%d\n", + __func__, + (int)prepcblk->ccp_rtcode, + (int)prepcblk->ccp_rscode); rc = -EIO; goto out; } @@ -1073,8 +1063,8 @@ static int _ip_cprb_helper(u16 cardnr, u16 domain, /* do some plausibility checks on the key block */ if (prepparm->kb.len < 120 + 3 * sizeof(uint16_t) || prepparm->kb.len > 136 + 3 * sizeof(uint16_t)) { - DEBUG_ERR("%s reply with invalid or unknown key block\n", - __func__); + ZCRYPT_DBF_ERR("%s reply with invalid or unknown key block\n", + __func__); rc = -EIO; goto out; } @@ -1132,33 +1122,29 @@ int cca_clr2cipherkey(u16 card, u16 dom, u32 keybitsize, u32 keygenflags, rc = _ip_cprb_helper(card, dom, "AES ", "FIRST ", "MIN3PART", exorbuf, keybitsize, token, &tokensize); if (rc) { - DEBUG_ERR( - "%s clear key import 1/4 with CSNBKPI2 failed, rc=%d\n", - __func__, rc); + ZCRYPT_DBF_ERR("%s clear key import 1/4 with CSNBKPI2 failed, rc=%d\n", + __func__, rc); goto out; } rc = _ip_cprb_helper(card, dom, "AES ", "ADD-PART", NULL, clrkey, keybitsize, token, &tokensize); if (rc) { - DEBUG_ERR( - "%s clear key import 2/4 with CSNBKPI2 failed, rc=%d\n", - __func__, rc); + ZCRYPT_DBF_ERR("%s clear key import 2/4 with CSNBKPI2 failed, rc=%d\n", + __func__, rc); goto out; } rc = _ip_cprb_helper(card, dom, "AES ", "ADD-PART", NULL, exorbuf, keybitsize, token, &tokensize); if (rc) { - DEBUG_ERR( - "%s clear key import 3/4 with CSNBKPI2 failed, rc=%d\n", - __func__, rc); + ZCRYPT_DBF_ERR("%s clear key import 3/4 with CSNBKPI2 failed, rc=%d\n", + __func__, rc); goto out; } rc = _ip_cprb_helper(card, dom, "AES ", "COMPLETE", NULL, NULL, keybitsize, token, &tokensize); if (rc) { - DEBUG_ERR( - "%s clear key import 4/4 with CSNBKPI2 failed, rc=%d\n", - __func__, rc); + ZCRYPT_DBF_ERR("%s clear key import 4/4 with CSNBKPI2 failed, rc=%d\n", + __func__, rc); goto out; } @@ -1265,19 +1251,17 @@ int cca_cipher2protkey(u16 cardnr, u16 domain, const u8 *ckey, /* forward xcrb with request CPRB and reply CPRB to zcrypt dd */ rc = zcrypt_send_cprb(&xcrb); if (rc) { - DEBUG_ERR( - "%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n", - __func__, (int)cardnr, (int)domain, rc); + ZCRYPT_DBF_ERR("%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n", + __func__, (int)cardnr, (int)domain, rc); goto out; } /* check response returncode and reasoncode */ if (prepcblk->ccp_rtcode != 0) { - DEBUG_ERR( - "%s unwrap secure key failure, card response %d/%d\n", - __func__, - (int)prepcblk->ccp_rtcode, - (int)prepcblk->ccp_rscode); + ZCRYPT_DBF_ERR("%s unwrap secure key failure, card response %d/%d\n", + __func__, + (int)prepcblk->ccp_rtcode, + (int)prepcblk->ccp_rscode); if (prepcblk->ccp_rtcode == 8 && prepcblk->ccp_rscode == 2290) rc = -EAGAIN; else @@ -1285,11 +1269,10 @@ int cca_cipher2protkey(u16 cardnr, u16 domain, const u8 *ckey, goto out; } if (prepcblk->ccp_rscode != 0) { - DEBUG_WARN( - "%s unwrap secure key warning, card response %d/%d\n", - __func__, - (int)prepcblk->ccp_rtcode, - (int)prepcblk->ccp_rscode); + ZCRYPT_DBF_WARN("%s unwrap secure key warning, card response %d/%d\n", + __func__, + (int)prepcblk->ccp_rtcode, + (int)prepcblk->ccp_rscode); } /* process response cprb param block */ @@ -1300,15 +1283,14 @@ int cca_cipher2protkey(u16 cardnr, u16 domain, const u8 *ckey, /* check the returned keyblock */ if (prepparm->vud.ckb.version != 0x01 && prepparm->vud.ckb.version != 0x02) { - DEBUG_ERR("%s reply param keyblock version mismatch 0x%02x\n", - __func__, (int)prepparm->vud.ckb.version); + ZCRYPT_DBF_ERR("%s reply param keyblock version mismatch 0x%02x\n", + __func__, (int)prepparm->vud.ckb.version); rc = -EIO; goto out; } if (prepparm->vud.ckb.algo != 0x02) { - DEBUG_ERR( - "%s reply param keyblock algo mismatch 0x%02x != 0x02\n", - __func__, (int)prepparm->vud.ckb.algo); + ZCRYPT_DBF_ERR("%s reply param keyblock algo mismatch 0x%02x != 0x02\n", + __func__, (int)prepparm->vud.ckb.algo); rc = -EIO; goto out; } @@ -1331,8 +1313,8 @@ int cca_cipher2protkey(u16 cardnr, u16 domain, const u8 *ckey, *protkeytype = PKEY_KEYTYPE_AES_256; break; default: - DEBUG_ERR("%s unknown/unsupported keylen %d\n", - __func__, prepparm->vud.ckb.keylen); + ZCRYPT_DBF_ERR("%s unknown/unsupported keylen %d\n", + __func__, prepparm->vud.ckb.keylen); rc = -EIO; goto out; } @@ -1432,19 +1414,17 @@ int cca_ecc2protkey(u16 cardnr, u16 domain, const u8 *key, /* forward xcrb with request CPRB and reply CPRB to zcrypt dd */ rc = zcrypt_send_cprb(&xcrb); if (rc) { - DEBUG_ERR( - "%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n", - __func__, (int)cardnr, (int)domain, rc); + ZCRYPT_DBF_ERR("%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n", + __func__, (int)cardnr, (int)domain, rc); goto out; } /* check response returncode and reasoncode */ if (prepcblk->ccp_rtcode != 0) { - DEBUG_ERR( - "%s unwrap secure key failure, card response %d/%d\n", - __func__, - (int)prepcblk->ccp_rtcode, - (int)prepcblk->ccp_rscode); + ZCRYPT_DBF_ERR("%s unwrap secure key failure, card response %d/%d\n", + __func__, + (int)prepcblk->ccp_rtcode, + (int)prepcblk->ccp_rscode); if (prepcblk->ccp_rtcode == 8 && prepcblk->ccp_rscode == 2290) rc = -EAGAIN; else @@ -1452,11 +1432,10 @@ int cca_ecc2protkey(u16 cardnr, u16 domain, const u8 *key, goto out; } if (prepcblk->ccp_rscode != 0) { - DEBUG_WARN( - "%s unwrap secure key warning, card response %d/%d\n", - __func__, - (int)prepcblk->ccp_rtcode, - (int)prepcblk->ccp_rscode); + ZCRYPT_DBF_WARN("%s unwrap secure key warning, card response %d/%d\n", + __func__, + (int)prepcblk->ccp_rtcode, + (int)prepcblk->ccp_rscode); } /* process response cprb param block */ @@ -1466,23 +1445,22 @@ int cca_ecc2protkey(u16 cardnr, u16 domain, const u8 *key, /* check the returned keyblock */ if (prepparm->vud.ckb.version != 0x02) { - DEBUG_ERR("%s reply param keyblock version mismatch 0x%02x != 0x02\n", - __func__, (int)prepparm->vud.ckb.version); + ZCRYPT_DBF_ERR("%s reply param keyblock version mismatch 0x%02x != 0x02\n", + __func__, (int)prepparm->vud.ckb.version); rc = -EIO; goto out; } if (prepparm->vud.ckb.algo != 0x81) { - DEBUG_ERR( - "%s reply param keyblock algo mismatch 0x%02x != 0x81\n", - __func__, (int)prepparm->vud.ckb.algo); + ZCRYPT_DBF_ERR("%s reply param keyblock algo mismatch 0x%02x != 0x81\n", + __func__, (int)prepparm->vud.ckb.algo); rc = -EIO; goto out; } /* copy the translated protected key */ if (prepparm->vud.ckb.keylen > *protkeylen) { - DEBUG_ERR("%s prot keylen mismatch %d > buffersize %u\n", - __func__, prepparm->vud.ckb.keylen, *protkeylen); + ZCRYPT_DBF_ERR("%s prot keylen mismatch %d > buffersize %u\n", + __func__, prepparm->vud.ckb.keylen, *protkeylen); rc = -EIO; goto out; } @@ -1550,17 +1528,17 @@ int cca_query_crypto_facility(u16 cardnr, u16 domain, /* forward xcrb with request CPRB and reply CPRB to zcrypt dd */ rc = zcrypt_send_cprb(&xcrb); if (rc) { - DEBUG_ERR("%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n", - __func__, (int)cardnr, (int)domain, rc); + ZCRYPT_DBF_ERR("%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n", + __func__, (int)cardnr, (int)domain, rc); goto out; } /* check response returncode and reasoncode */ if (prepcblk->ccp_rtcode != 0) { - DEBUG_ERR("%s unwrap secure key failure, card response %d/%d\n", - __func__, - (int)prepcblk->ccp_rtcode, - (int)prepcblk->ccp_rscode); + ZCRYPT_DBF_ERR("%s unwrap secure key failure, card response %d/%d\n", + __func__, + (int)prepcblk->ccp_rtcode, + (int)prepcblk->ccp_rscode); rc = -EIO; goto out; } diff --git a/drivers/s390/crypto/zcrypt_debug.h b/drivers/s390/crypto/zcrypt_debug.h index 5cf88aabd64b..9a208dc4c200 100644 --- a/drivers/s390/crypto/zcrypt_debug.h +++ b/drivers/s390/crypto/zcrypt_debug.h @@ -17,7 +17,7 @@ #define RC2ERR(rc) ((rc) ? DBF_ERR : DBF_INFO) #define RC2WARN(rc) ((rc) ? DBF_WARN : DBF_INFO) -#define DBF_MAX_SPRINTF_ARGS 6 +#define ZCRYPT_DBF_MAX_SPRINTF_ARGS 6 #define ZCRYPT_DBF(...) \ debug_sprintf_event(zcrypt_dbf_info, ##__VA_ARGS__) @@ -27,8 +27,6 @@ debug_sprintf_event(zcrypt_dbf_info, DBF_WARN, ##__VA_ARGS__) #define ZCRYPT_DBF_INFO(...) \ debug_sprintf_event(zcrypt_dbf_info, DBF_INFO, ##__VA_ARGS__) -#define ZCRYPT_DBF_DBG(...) \ - debug_sprintf_event(zcrypt_dbf_info, DBF_DEBUG, ##__VA_ARGS__) extern debug_info_t *zcrypt_dbf_info; diff --git a/drivers/s390/crypto/zcrypt_ep11misc.c b/drivers/s390/crypto/zcrypt_ep11misc.c index 0a877f9792c2..eb7f5489ccf9 100644 --- a/drivers/s390/crypto/zcrypt_ep11misc.c +++ b/drivers/s390/crypto/zcrypt_ep11misc.c @@ -24,11 +24,6 @@ #include "zcrypt_ep11misc.h" #include "zcrypt_ccamisc.h" -#define DEBUG_DBG(...) ZCRYPT_DBF(DBF_DEBUG, ##__VA_ARGS__) -#define DEBUG_INFO(...) ZCRYPT_DBF(DBF_INFO, ##__VA_ARGS__) -#define DEBUG_WARN(...) ZCRYPT_DBF(DBF_WARN, ##__VA_ARGS__) -#define DEBUG_ERR(...) ZCRYPT_DBF(DBF_ERR, ##__VA_ARGS__) - #define EP11_PINBLOB_V1_BYTES 56 /* default iv used here */ @@ -510,7 +505,7 @@ static int check_reply_pl(const u8 *pl, const char *func) /* start tag */ if (*pl++ != 0x30) { - DEBUG_ERR("%s reply start tag mismatch\n", func); + ZCRYPT_DBF_ERR("%s reply start tag mismatch\n", func); return -EIO; } @@ -527,40 +522,41 @@ static int check_reply_pl(const u8 *pl, const char *func) len = *((u16 *)pl); pl += 2; } else { - DEBUG_ERR("%s reply start tag lenfmt mismatch 0x%02hhx\n", - func, *pl); + ZCRYPT_DBF_ERR("%s reply start tag lenfmt mismatch 0x%02hhx\n", + func, *pl); return -EIO; } /* len should cover at least 3 fields with 32 bit value each */ if (len < 3 * 6) { - DEBUG_ERR("%s reply length %d too small\n", func, len); + ZCRYPT_DBF_ERR("%s reply length %d too small\n", func, len); return -EIO; } /* function tag, length and value */ if (pl[0] != 0x04 || pl[1] != 0x04) { - DEBUG_ERR("%s function tag or length mismatch\n", func); + ZCRYPT_DBF_ERR("%s function tag or length mismatch\n", func); return -EIO; } pl += 6; /* dom tag, length and value */ if (pl[0] != 0x04 || pl[1] != 0x04) { - DEBUG_ERR("%s dom tag or length mismatch\n", func); + ZCRYPT_DBF_ERR("%s dom tag or length mismatch\n", func); return -EIO; } pl += 6; /* return value tag, length and value */ if (pl[0] != 0x04 || pl[1] != 0x04) { - DEBUG_ERR("%s return value tag or length mismatch\n", func); + ZCRYPT_DBF_ERR("%s return value tag or length mismatch\n", + func); return -EIO; } pl += 2; ret = *((u32 *)pl); if (ret != 0) { - DEBUG_ERR("%s return value 0x%04x != 0\n", func, ret); + ZCRYPT_DBF_ERR("%s return value 0x%04x != 0\n", func, ret); return -EIO; } @@ -626,9 +622,8 @@ static int ep11_query_info(u16 cardnr, u16 domain, u32 query_type, rc = zcrypt_send_ep11_cprb(urb); if (rc) { - DEBUG_ERR( - "%s zcrypt_send_ep11_cprb(card=%d dom=%d) failed, rc=%d\n", - __func__, (int)cardnr, (int)domain, rc); + ZCRYPT_DBF_ERR("%s zcrypt_send_ep11_cprb(card=%d dom=%d) failed, rc=%d\n", + __func__, (int)cardnr, (int)domain, rc); goto out; } @@ -636,13 +631,13 @@ static int ep11_query_info(u16 cardnr, u16 domain, u32 query_type, if (rc) goto out; if (rep_pl->data_tag != 0x04 || rep_pl->data_lenfmt != 0x82) { - DEBUG_ERR("%s unknown reply data format\n", __func__); + ZCRYPT_DBF_ERR("%s unknown reply data format\n", __func__); rc = -EIO; goto out; } if (rep_pl->data_len > buflen) { - DEBUG_ERR("%s mismatch between reply data len and buffer len\n", - __func__); + ZCRYPT_DBF_ERR("%s mismatch between reply data len and buffer len\n", + __func__); rc = -ENOSPC; goto out; } @@ -816,9 +811,8 @@ static int _ep11_genaeskey(u16 card, u16 domain, case 256: break; default: - DEBUG_ERR( - "%s unknown/unsupported keybitsize %d\n", - __func__, keybitsize); + ZCRYPT_DBF_ERR("%s unknown/unsupported keybitsize %d\n", + __func__, keybitsize); rc = -EINVAL; goto out; } @@ -878,9 +872,8 @@ static int _ep11_genaeskey(u16 card, u16 domain, rc = zcrypt_send_ep11_cprb(urb); if (rc) { - DEBUG_ERR( - "%s zcrypt_send_ep11_cprb(card=%d dom=%d) failed, rc=%d\n", - __func__, (int)card, (int)domain, rc); + ZCRYPT_DBF_ERR("%s zcrypt_send_ep11_cprb(card=%d dom=%d) failed, rc=%d\n", + __func__, (int)card, (int)domain, rc); goto out; } @@ -888,13 +881,13 @@ static int _ep11_genaeskey(u16 card, u16 domain, if (rc) goto out; if (rep_pl->data_tag != 0x04 || rep_pl->data_lenfmt != 0x82) { - DEBUG_ERR("%s unknown reply data format\n", __func__); + ZCRYPT_DBF_ERR("%s unknown reply data format\n", __func__); rc = -EIO; goto out; } if (rep_pl->data_len > *keybufsize) { - DEBUG_ERR("%s mismatch reply data len / key buffer len\n", - __func__); + ZCRYPT_DBF_ERR("%s mismatch reply data len / key buffer len\n", + __func__); rc = -ENOSPC; goto out; } @@ -1030,9 +1023,8 @@ static int ep11_cryptsingle(u16 card, u16 domain, rc = zcrypt_send_ep11_cprb(urb); if (rc) { - DEBUG_ERR( - "%s zcrypt_send_ep11_cprb(card=%d dom=%d) failed, rc=%d\n", - __func__, (int)card, (int)domain, rc); + ZCRYPT_DBF_ERR("%s zcrypt_send_ep11_cprb(card=%d dom=%d) failed, rc=%d\n", + __func__, (int)card, (int)domain, rc); goto out; } @@ -1040,7 +1032,7 @@ static int ep11_cryptsingle(u16 card, u16 domain, if (rc) goto out; if (rep_pl->data_tag != 0x04) { - DEBUG_ERR("%s unknown reply data format\n", __func__); + ZCRYPT_DBF_ERR("%s unknown reply data format\n", __func__); rc = -EIO; goto out; } @@ -1053,14 +1045,14 @@ static int ep11_cryptsingle(u16 card, u16 domain, n = *((u16 *)p); p += 2; } else { - DEBUG_ERR("%s unknown reply data length format 0x%02hhx\n", - __func__, rep_pl->data_lenfmt); + ZCRYPT_DBF_ERR("%s unknown reply data length format 0x%02hhx\n", + __func__, rep_pl->data_lenfmt); rc = -EIO; goto out; } if (n > *outbufsize) { - DEBUG_ERR("%s mismatch reply data len %d / output buffer %zu\n", - __func__, n, *outbufsize); + ZCRYPT_DBF_ERR("%s mismatch reply data len %d / output buffer %zu\n", + __func__, n, *outbufsize); rc = -ENOSPC; goto out; } @@ -1188,9 +1180,8 @@ static int _ep11_unwrapkey(u16 card, u16 domain, rc = zcrypt_send_ep11_cprb(urb); if (rc) { - DEBUG_ERR( - "%s zcrypt_send_ep11_cprb(card=%d dom=%d) failed, rc=%d\n", - __func__, (int)card, (int)domain, rc); + ZCRYPT_DBF_ERR("%s zcrypt_send_ep11_cprb(card=%d dom=%d) failed, rc=%d\n", + __func__, (int)card, (int)domain, rc); goto out; } @@ -1198,13 +1189,13 @@ static int _ep11_unwrapkey(u16 card, u16 domain, if (rc) goto out; if (rep_pl->data_tag != 0x04 || rep_pl->data_lenfmt != 0x82) { - DEBUG_ERR("%s unknown reply data format\n", __func__); + ZCRYPT_DBF_ERR("%s unknown reply data format\n", __func__); rc = -EIO; goto out; } if (rep_pl->data_len > *keybufsize) { - DEBUG_ERR("%s mismatch reply data len / key buffer len\n", - __func__); + ZCRYPT_DBF_ERR("%s mismatch reply data len / key buffer len\n", + __func__); rc = -ENOSPC; goto out; } @@ -1343,9 +1334,8 @@ static int _ep11_wrapkey(u16 card, u16 domain, rc = zcrypt_send_ep11_cprb(urb); if (rc) { - DEBUG_ERR( - "%s zcrypt_send_ep11_cprb(card=%d dom=%d) failed, rc=%d\n", - __func__, (int)card, (int)domain, rc); + ZCRYPT_DBF_ERR("%s zcrypt_send_ep11_cprb(card=%d dom=%d) failed, rc=%d\n", + __func__, (int)card, (int)domain, rc); goto out; } @@ -1353,13 +1343,13 @@ static int _ep11_wrapkey(u16 card, u16 domain, if (rc) goto out; if (rep_pl->data_tag != 0x04 || rep_pl->data_lenfmt != 0x82) { - DEBUG_ERR("%s unknown reply data format\n", __func__); + ZCRYPT_DBF_ERR("%s unknown reply data format\n", __func__); rc = -EIO; goto out; } if (rep_pl->data_len > *datasize) { - DEBUG_ERR("%s mismatch reply data len / data buffer len\n", - __func__); + ZCRYPT_DBF_ERR("%s mismatch reply data len / data buffer len\n", + __func__); rc = -ENOSPC; goto out; } @@ -1386,9 +1376,8 @@ int ep11_clr2keyblob(u16 card, u16 domain, u32 keybitsize, u32 keygenflags, if (keybitsize == 128 || keybitsize == 192 || keybitsize == 256) { clrkeylen = keybitsize / 8; } else { - DEBUG_ERR( - "%s unknown/unsupported keybitsize %d\n", - __func__, keybitsize); + ZCRYPT_DBF_ERR("%s unknown/unsupported keybitsize %d\n", + __func__, keybitsize); return -EINVAL; } @@ -1405,9 +1394,8 @@ int ep11_clr2keyblob(u16 card, u16 domain, u32 keybitsize, u32 keygenflags, 0x00006c00, /* EN/DECRYPT, WRAP/UNWRAP */ kek, &keklen); if (rc) { - DEBUG_ERR( - "%s generate kek key failed, rc=%d\n", - __func__, rc); + ZCRYPT_DBF_ERR("%s generate kek key failed, rc=%d\n", + __func__, rc); goto out; } @@ -1415,9 +1403,8 @@ int ep11_clr2keyblob(u16 card, u16 domain, u32 keybitsize, u32 keygenflags, rc = ep11_cryptsingle(card, domain, 0, 0, def_iv, kek, keklen, clrkey, clrkeylen, encbuf, &encbuflen); if (rc) { - DEBUG_ERR( - "%s encrypting key value with kek key failed, rc=%d\n", - __func__, rc); + ZCRYPT_DBF_ERR("%s encrypting key value with kek key failed, rc=%d\n", + __func__, rc); goto out; } @@ -1426,9 +1413,8 @@ int ep11_clr2keyblob(u16 card, u16 domain, u32 keybitsize, u32 keygenflags, encbuf, encbuflen, 0, def_iv, keybitsize, 0, keybuf, keybufsize, keytype); if (rc) { - DEBUG_ERR( - "%s importing key value as new key failed,, rc=%d\n", - __func__, rc); + ZCRYPT_DBF_ERR("%s importing key value as new key failed,, rc=%d\n", + __func__, rc); goto out; } @@ -1476,17 +1462,16 @@ int ep11_kblob2protkey(u16 card, u16 dom, rc = _ep11_wrapkey(card, dom, (u8 *)key, keylen, 0, def_iv, wkbuf, &wkbuflen); if (rc) { - DEBUG_ERR( - "%s rewrapping ep11 key to pkey failed, rc=%d\n", - __func__, rc); + ZCRYPT_DBF_ERR("%s rewrapping ep11 key to pkey failed, rc=%d\n", + __func__, rc); goto out; } wki = (struct wk_info *)wkbuf; /* check struct version and pkey type */ if (wki->version != 1 || wki->pkeytype < 1 || wki->pkeytype > 5) { - DEBUG_ERR("%s wk info version %d or pkeytype %d mismatch.\n", - __func__, (int)wki->version, (int)wki->pkeytype); + ZCRYPT_DBF_ERR("%s wk info version %d or pkeytype %d mismatch.\n", + __func__, (int)wki->version, (int)wki->pkeytype); rc = -EIO; goto out; } @@ -1511,8 +1496,8 @@ int ep11_kblob2protkey(u16 card, u16 dom, *protkeytype = PKEY_KEYTYPE_AES_256; break; default: - DEBUG_ERR("%s unknown/unsupported AES pkeysize %d\n", - __func__, (int)wki->pkeysize); + ZCRYPT_DBF_ERR("%s unknown/unsupported AES pkeysize %d\n", + __func__, (int)wki->pkeysize); rc = -EIO; goto out; } @@ -1525,16 +1510,16 @@ int ep11_kblob2protkey(u16 card, u16 dom, break; case 2: /* TDES */ default: - DEBUG_ERR("%s unknown/unsupported key type %d\n", - __func__, (int)wki->pkeytype); + ZCRYPT_DBF_ERR("%s unknown/unsupported key type %d\n", + __func__, (int)wki->pkeytype); rc = -EIO; goto out; } /* copy the translated protected key */ if (wki->pkeysize > *protkeylen) { - DEBUG_ERR("%s wk info pkeysize %llu > protkeysize %u\n", - __func__, wki->pkeysize, *protkeylen); + ZCRYPT_DBF_ERR("%s wk info pkeysize %llu > protkeysize %u\n", + __func__, wki->pkeysize, *protkeylen); rc = -EINVAL; goto out; } diff --git a/drivers/s390/crypto/zcrypt_error.h b/drivers/s390/crypto/zcrypt_error.h index a44fcfcec938..46e27b43a8af 100644 --- a/drivers/s390/crypto/zcrypt_error.h +++ b/drivers/s390/crypto/zcrypt_error.h @@ -119,10 +119,9 @@ static inline int convert_error(struct zcrypt_queue *zq, case REP82_ERROR_MESSAGE_TYPE: /* 0x20 */ case REP82_ERROR_TRANSPORT_FAIL: /* 0x90 */ /* - * Msg to wrong type or card/infrastructure failure. - * Trigger rescan of the ap bus, trigger retry request. + * Msg to wrong type or card/infrastructure failure. Return + * EAGAIN, the upper layer may do a retry on the request. */ - atomic_set(&zcrypt_rescan_req, 1); /* For type 86 response show the apfs value (failure reason) */ if (ehdr->reply_code == REP82_ERROR_TRANSPORT_FAIL && ehdr->type == TYPE86_RSP_CODE) { diff --git a/drivers/s390/crypto/zcrypt_msgtype50.c b/drivers/s390/crypto/zcrypt_msgtype50.c index 2e155de8abe5..3b39cb8f926d 100644 --- a/drivers/s390/crypto/zcrypt_msgtype50.c +++ b/drivers/s390/crypto/zcrypt_msgtype50.c @@ -427,7 +427,7 @@ static void zcrypt_msgtype50_receive(struct ap_queue *aq, len = t80h->len; if (len > reply->bufsize || len > msg->bufsize || len != reply->len) { - ZCRYPT_DBF_DBG("%s len mismatch => EMSGSIZE\n", __func__); + pr_debug("%s len mismatch => EMSGSIZE\n", __func__); msg->rc = -EMSGSIZE; goto out; } @@ -487,9 +487,9 @@ static long zcrypt_msgtype50_modexpo(struct zcrypt_queue *zq, out: ap_msg->private = NULL; if (rc) - ZCRYPT_DBF_DBG("%s send me cprb at dev=%02x.%04x rc=%d\n", - __func__, AP_QID_CARD(zq->queue->qid), - AP_QID_QUEUE(zq->queue->qid), rc); + pr_debug("%s send me cprb at dev=%02x.%04x rc=%d\n", + __func__, AP_QID_CARD(zq->queue->qid), + AP_QID_QUEUE(zq->queue->qid), rc); return rc; } @@ -537,9 +537,9 @@ static long zcrypt_msgtype50_modexpo_crt(struct zcrypt_queue *zq, out: ap_msg->private = NULL; if (rc) - ZCRYPT_DBF_DBG("%s send crt cprb at dev=%02x.%04x rc=%d\n", - __func__, AP_QID_CARD(zq->queue->qid), - AP_QID_QUEUE(zq->queue->qid), rc); + pr_debug("%s send crt cprb at dev=%02x.%04x rc=%d\n", + __func__, AP_QID_CARD(zq->queue->qid), + AP_QID_QUEUE(zq->queue->qid), rc); return rc; } diff --git a/drivers/s390/crypto/zcrypt_msgtype6.c b/drivers/s390/crypto/zcrypt_msgtype6.c index 3c53abbdc342..215f257d2360 100644 --- a/drivers/s390/crypto/zcrypt_msgtype6.c +++ b/drivers/s390/crypto/zcrypt_msgtype6.c @@ -437,9 +437,9 @@ static int xcrb_msg_to_type6cprb_msgx(bool userspace, struct ap_message *ap_msg, ap_msg->flags |= AP_MSG_FLAG_ADMIN; break; default: - ZCRYPT_DBF_DBG("%s unknown CPRB minor version '%c%c'\n", - __func__, msg->cprbx.func_id[0], - msg->cprbx.func_id[1]); + pr_debug("%s unknown CPRB minor version '%c%c'\n", + __func__, msg->cprbx.func_id[0], + msg->cprbx.func_id[1]); } /* copy data block */ @@ -629,9 +629,9 @@ static int convert_type86_xcrb(bool userspace, struct zcrypt_queue *zq, /* Copy CPRB to user */ if (xcrb->reply_control_blk_length < msg->fmt2.count1) { - ZCRYPT_DBF_DBG("%s reply_control_blk_length %u < required %u => EMSGSIZE\n", - __func__, xcrb->reply_control_blk_length, - msg->fmt2.count1); + pr_debug("%s reply_control_blk_length %u < required %u => EMSGSIZE\n", + __func__, xcrb->reply_control_blk_length, + msg->fmt2.count1); return -EMSGSIZE; } if (z_copy_to_user(userspace, xcrb->reply_control_blk_addr, @@ -642,9 +642,9 @@ static int convert_type86_xcrb(bool userspace, struct zcrypt_queue *zq, /* Copy data buffer to user */ if (msg->fmt2.count2) { if (xcrb->reply_data_length < msg->fmt2.count2) { - ZCRYPT_DBF_DBG("%s reply_data_length %u < required %u => EMSGSIZE\n", - __func__, xcrb->reply_data_length, - msg->fmt2.count2); + pr_debug("%s reply_data_length %u < required %u => EMSGSIZE\n", + __func__, xcrb->reply_data_length, + msg->fmt2.count2); return -EMSGSIZE; } if (z_copy_to_user(userspace, xcrb->reply_data_addr, @@ -673,9 +673,9 @@ static int convert_type86_ep11_xcrb(bool userspace, struct zcrypt_queue *zq, char *data = reply->msg; if (xcrb->resp_len < msg->fmt2.count1) { - ZCRYPT_DBF_DBG("%s resp_len %u < required %u => EMSGSIZE\n", - __func__, (unsigned int)xcrb->resp_len, - msg->fmt2.count1); + pr_debug("%s resp_len %u < required %u => EMSGSIZE\n", + __func__, (unsigned int)xcrb->resp_len, + msg->fmt2.count1); return -EMSGSIZE; } @@ -875,7 +875,8 @@ static void zcrypt_msgtype6_receive(struct ap_queue *aq, len = sizeof(struct type86x_reply) + t86r->length; if (len > reply->bufsize || len > msg->bufsize || len != reply->len) { - ZCRYPT_DBF_DBG("%s len mismatch => EMSGSIZE\n", __func__); + pr_debug("%s len mismatch => EMSGSIZE\n", + __func__); msg->rc = -EMSGSIZE; goto out; } @@ -889,7 +890,8 @@ static void zcrypt_msgtype6_receive(struct ap_queue *aq, len = t86r->fmt2.offset1 + t86r->fmt2.count1; if (len > reply->bufsize || len > msg->bufsize || len != reply->len) { - ZCRYPT_DBF_DBG("%s len mismatch => EMSGSIZE\n", __func__); + pr_debug("%s len mismatch => EMSGSIZE\n", + __func__); msg->rc = -EMSGSIZE; goto out; } @@ -939,7 +941,8 @@ static void zcrypt_msgtype6_receive_ep11(struct ap_queue *aq, len = t86r->fmt2.offset1 + t86r->fmt2.count1; if (len > reply->bufsize || len > msg->bufsize || len != reply->len) { - ZCRYPT_DBF_DBG("%s len mismatch => EMSGSIZE\n", __func__); + pr_debug("%s len mismatch => EMSGSIZE\n", + __func__); msg->rc = -EMSGSIZE; goto out; } @@ -1151,9 +1154,9 @@ static long zcrypt_msgtype6_send_cprb(bool userspace, struct zcrypt_queue *zq, out: if (rc) - ZCRYPT_DBF_DBG("%s send cprb at dev=%02x.%04x rc=%d\n", - __func__, AP_QID_CARD(zq->queue->qid), - AP_QID_QUEUE(zq->queue->qid), rc); + pr_debug("%s send cprb at dev=%02x.%04x rc=%d\n", + __func__, AP_QID_CARD(zq->queue->qid), + AP_QID_QUEUE(zq->queue->qid), rc); return rc; } @@ -1274,9 +1277,9 @@ static long zcrypt_msgtype6_send_ep11_cprb(bool userspace, struct zcrypt_queue * out: if (rc) - ZCRYPT_DBF_DBG("%s send cprb at dev=%02x.%04x rc=%d\n", - __func__, AP_QID_CARD(zq->queue->qid), - AP_QID_QUEUE(zq->queue->qid), rc); + pr_debug("%s send cprb at dev=%02x.%04x rc=%d\n", + __func__, AP_QID_CARD(zq->queue->qid), + AP_QID_QUEUE(zq->queue->qid), rc); return rc; } diff --git a/include/linux/compiler_attributes.h b/include/linux/compiler_attributes.h index 289810685fc5..8bdf6e0918c1 100644 --- a/include/linux/compiler_attributes.h +++ b/include/linux/compiler_attributes.h @@ -334,6 +334,18 @@ #define __section(section) __attribute__((__section__(section))) /* + * Optional: only supported since gcc >= 12 + * + * gcc: https://gcc.gnu.org/onlinedocs/gcc/Common-Variable-Attributes.html#index-uninitialized-variable-attribute + * clang: https://clang.llvm.org/docs/AttributeReference.html#uninitialized + */ +#if __has_attribute(__uninitialized__) +# define __uninitialized __attribute__((__uninitialized__)) +#else +# define __uninitialized +#endif + +/* * gcc: https://gcc.gnu.org/onlinedocs/gcc/Common-Function-Attributes.html#index-unused-function-attribute * gcc: https://gcc.gnu.org/onlinedocs/gcc/Common-Type-Attributes.html#index-unused-type-attribute * gcc: https://gcc.gnu.org/onlinedocs/gcc/Common-Variable-Attributes.html#index-unused-variable-attribute diff --git a/lib/raid6/s390vx.uc b/lib/raid6/s390vx.uc index cd0b9e95f499..863e2d320938 100644 --- a/lib/raid6/s390vx.uc +++ b/lib/raid6/s390vx.uc @@ -12,15 +12,14 @@ */ #include <linux/raid/pq.h> -#include <asm/fpu/api.h> -#include <asm/vx-insn.h> +#include <asm/fpu.h> #define NSIZE 16 -static inline void LOAD_CONST(void) +static __always_inline void LOAD_CONST(void) { - asm volatile("VREPIB %v24,7"); - asm volatile("VREPIB %v25,0x1d"); + fpu_vrepib(24, 0x07); + fpu_vrepib(25, 0x1d); } /* @@ -28,10 +27,7 @@ static inline void LOAD_CONST(void) * vector register y left by 1 bit and stores the result in * vector register x. */ -static inline void SHLBYTE(int x, int y) -{ - asm volatile ("VAB %0,%1,%1" : : "i" (x), "i" (y)); -} +#define SHLBYTE(x, y) fpu_vab(x, y, y) /* * For each of the 16 bytes in the vector register y the MASK() @@ -39,49 +35,17 @@ static inline void SHLBYTE(int x, int y) * or 0x00 if the high bit is 0. The result is stored in vector * register x. */ -static inline void MASK(int x, int y) -{ - asm volatile ("VESRAVB %0,%1,24" : : "i" (x), "i" (y)); -} - -static inline void AND(int x, int y, int z) -{ - asm volatile ("VN %0,%1,%2" : : "i" (x), "i" (y), "i" (z)); -} - -static inline void XOR(int x, int y, int z) -{ - asm volatile ("VX %0,%1,%2" : : "i" (x), "i" (y), "i" (z)); -} +#define MASK(x, y) fpu_vesravb(x, y, 24) -static inline void LOAD_DATA(int x, u8 *ptr) -{ - typedef struct { u8 _[16 * $#]; } addrtype; - register addrtype *__ptr asm("1") = (addrtype *) ptr; - - asm volatile ("VLM %2,%3,0,%1" - : : "m" (*__ptr), "a" (__ptr), "i" (x), - "i" (x + $# - 1)); -} - -static inline void STORE_DATA(int x, u8 *ptr) -{ - typedef struct { u8 _[16 * $#]; } addrtype; - register addrtype *__ptr asm("1") = (addrtype *) ptr; - - asm volatile ("VSTM %2,%3,0,1" - : "=m" (*__ptr) : "a" (__ptr), "i" (x), - "i" (x + $# - 1)); -} - -static inline void COPY_VEC(int x, int y) -{ - asm volatile ("VLR %0,%1" : : "i" (x), "i" (y)); -} +#define AND(x, y, z) fpu_vn(x, y, z) +#define XOR(x, y, z) fpu_vx(x, y, z) +#define LOAD_DATA(x, ptr) fpu_vlm(x, x + $# - 1, ptr) +#define STORE_DATA(x, ptr) fpu_vstm(x, x + $# - 1, ptr) +#define COPY_VEC(x, y) fpu_vlr(x, y) static void raid6_s390vx$#_gen_syndrome(int disks, size_t bytes, void **ptrs) { - struct kernel_fpu vxstate; + DECLARE_KERNEL_FPU_ONSTACK32(vxstate); u8 **dptr, *p, *q; int d, z, z0; @@ -114,7 +78,7 @@ static void raid6_s390vx$#_gen_syndrome(int disks, size_t bytes, void **ptrs) static void raid6_s390vx$#_xor_syndrome(int disks, int start, int stop, size_t bytes, void **ptrs) { - struct kernel_fpu vxstate; + DECLARE_KERNEL_FPU_ONSTACK32(vxstate); u8 **dptr, *p, *q; int d, z, z0; diff --git a/tools/testing/selftests/kvm/s390x/memop.c b/tools/testing/selftests/kvm/s390x/memop.c index bb3ca9a5d731..b6da8f71ea19 100644 --- a/tools/testing/selftests/kvm/s390x/memop.c +++ b/tools/testing/selftests/kvm/s390x/memop.c @@ -375,6 +375,32 @@ static void test_copy(void) kvm_vm_free(t.kvm_vm); } +static void test_copy_access_register(void) +{ + struct test_default t = test_default_init(guest_copy); + + HOST_SYNC(t.vcpu, STAGE_INITED); + + prepare_mem12(); + t.run->psw_mask &= ~(3UL << (63 - 17)); + t.run->psw_mask |= 1UL << (63 - 17); /* Enable AR mode */ + + /* + * Primary address space gets used if an access register + * contains zero. The host makes use of AR[1] so is a good + * candidate to ensure the guest AR (of zero) is used. + */ + CHECK_N_DO(MOP, t.vcpu, LOGICAL, WRITE, mem1, t.size, + GADDR_V(mem1), AR(1)); + HOST_SYNC(t.vcpu, STAGE_COPIED); + + CHECK_N_DO(MOP, t.vcpu, LOGICAL, READ, mem2, t.size, + GADDR_V(mem2), AR(1)); + ASSERT_MEM_EQ(mem1, mem2, t.size); + + kvm_vm_free(t.kvm_vm); +} + static void set_storage_key_range(void *addr, size_t len, uint8_t key) { uintptr_t _addr, abs, i; @@ -1102,6 +1128,11 @@ int main(int argc, char *argv[]) .requirements_met = extension_cap > 0, }, { + .name = "copy with access register mode", + .test = test_copy_access_register, + .requirements_met = true, + }, + { .name = "error checks with key", .test = test_errors_key, .requirements_met = extension_cap > 0, |